diff --git a/README.md b/README.md index 4f3965bb..a057102b 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,20 @@ - checks out the model versions specified by the user - builds the required executables -- runs each model version across N standard science configurations +- runs each model version across N standard science configurations for a variety of meteorological forcings - performs bitwise comparison checks on model outputs across model versions The user can then pipe the model outputs into a benchmark analysis via [modelevaluation.org][meorg] to assess model performance. The full documentation is available at [benchcab.readthedocs.io][docs]. +## Supported configurations + +`benchcab` currently tests the following model configurations for CABLE: + +- **Flux site simulations (offline)** - running CABLE forced with observed eddy covariance data at a single site +- **Global/regional simulations (offline)** - running CABLE forced with meteorological fields over a region (global or regional) + ## License `benchcab` is distributed under [an Apache License v2.0][apache-license]. diff --git a/benchcab/benchcab.py b/benchcab/benchcab.py index 141595da..05b0e331 100644 --- a/benchcab/benchcab.py +++ b/benchcab/benchcab.py @@ -10,17 +10,10 @@ from subprocess import CalledProcessError from typing import Optional -from benchcab import internal +from benchcab import fluxsite, internal, spatial from benchcab.comparison import run_comparisons, run_comparisons_in_parallel from benchcab.config import read_config from benchcab.environment_modules import EnvironmentModules, EnvironmentModulesInterface -from benchcab.fluxsite import ( - Task, - get_fluxsite_comparisons, - get_fluxsite_tasks, - run_tasks, - run_tasks_in_parallel, -) from benchcab.internal import get_met_forcing_file_names from benchcab.model import Model from benchcab.utils import get_logger @@ -28,7 +21,10 @@ from benchcab.utils.pbs import render_job_script from benchcab.utils.repo import create_repo from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface -from benchcab.workdir import setup_fluxsite_directory_tree +from benchcab.workdir import ( + setup_fluxsite_directory_tree, + setup_spatial_directory_tree, +) class Benchcab: @@ -57,7 +53,8 @@ def __init__( self._config: Optional[dict] = None self._models: list[Model] = [] - self.tasks: list[Task] = [] # initialise fluxsite tasks lazily + self._fluxsite_tasks: list[fluxsite.FluxsiteTask] = [] + self._spatial_tasks: list[spatial.SpatialTask] = [] # Get the logger object self.logger = get_logger() @@ -148,16 +145,26 @@ def _get_models(self, config: dict) -> list[Model]: self._models.append(Model(repo=repo, model_id=id, **sub_config)) return self._models - def _initialise_tasks(self, config: dict) -> list[Task]: - """A helper method that initialises and returns the `tasks` attribute.""" - self.tasks = get_fluxsite_tasks( - models=self._get_models(config), - science_configurations=config["science_configurations"], - fluxsite_forcing_file_names=get_met_forcing_file_names( - config["fluxsite"]["experiment"] - ), - ) - return self.tasks + def _get_fluxsite_tasks(self, config: dict) -> list[fluxsite.FluxsiteTask]: + if not self._fluxsite_tasks: + self._fluxsite_tasks = fluxsite.get_fluxsite_tasks( + models=self._get_models(config), + science_configurations=config["science_configurations"], + fluxsite_forcing_file_names=get_met_forcing_file_names( + config["fluxsite"]["experiment"] + ), + ) + return self._fluxsite_tasks + + def _get_spatial_tasks(self, config) -> list[spatial.SpatialTask]: + if not self._spatial_tasks: + self._spatial_tasks = spatial.get_spatial_tasks( + models=self._get_models(config), + met_forcings=config["spatial"]["met_forcings"], + science_configurations=config["science_configurations"], + payu_args=config["spatial"]["payu"]["args"], + ) + return self._spatial_tasks def validate_config(self, config_path: str): """Endpoint for `benchcab validate_config`.""" @@ -226,7 +233,7 @@ def checkout(self, config_path: str): with rev_number_log_path.open("w", encoding="utf-8") as file: file.write(rev_number_log) - def build(self, config_path: str): + def build(self, config_path: str, mpi=False): """Endpoint for `benchcab build`.""" config = self._get_config(config_path) self._validate_environment(project=config["project"], modules=config["modules"]) @@ -239,13 +246,13 @@ def build(self, config_path: str): repo.custom_build(modules=config["modules"]) else: - build_mode = "with MPI" if internal.MPI else "serially" + build_mode = "with MPI" if mpi else "serially" self.logger.info( f"Compiling CABLE {build_mode} for realisation {repo.name}..." ) - repo.pre_build() - repo.run_build(modules=config["modules"]) - repo.post_build() + repo.pre_build(mpi=mpi) + repo.run_build(modules=config["modules"], mpi=mpi) + repo.post_build(mpi=mpi) self.logger.info(f"Successfully compiled CABLE for realisation {repo.name}") def fluxsite_setup_work_directory(self, config_path: str): @@ -253,11 +260,10 @@ def fluxsite_setup_work_directory(self, config_path: str): config = self._get_config(config_path) self._validate_environment(project=config["project"], modules=config["modules"]) - tasks = self.tasks if self.tasks else self._initialise_tasks(config) self.logger.info("Setting up run directory tree for fluxsite tests...") setup_fluxsite_directory_tree() self.logger.info("Setting up tasks...") - for task in tasks: + for task in self._get_fluxsite_tasks(config): task.setup_task() self.logger.info("Successfully setup fluxsite tasks") @@ -265,14 +271,14 @@ def fluxsite_run_tasks(self, config_path: str): """Endpoint for `benchcab fluxsite-run-tasks`.""" config = self._get_config(config_path) self._validate_environment(project=config["project"], modules=config["modules"]) + tasks = self._get_fluxsite_tasks(config) - tasks = self.tasks if self.tasks else self._initialise_tasks(config) self.logger.info("Running fluxsite tasks...") if config["fluxsite"]["multiprocess"]: ncpus = config["fluxsite"]["pbs"]["ncpus"] - run_tasks_in_parallel(tasks, n_processes=ncpus) + fluxsite.run_tasks_in_parallel(tasks, n_processes=ncpus) else: - run_tasks(tasks) + fluxsite.run_tasks(tasks) self.logger.info("Successfully ran fluxsite tasks") def fluxsite_bitwise_cmp(self, config_path: str): @@ -285,8 +291,9 @@ def fluxsite_bitwise_cmp(self, config_path: str): "nccmp/1.8.5.0" ) # use `nccmp -df` for bitwise comparisons - tasks = self.tasks if self.tasks else self._initialise_tasks(config) - comparisons = get_fluxsite_comparisons(tasks) + comparisons = fluxsite.get_fluxsite_comparisons( + self._get_fluxsite_tasks(config) + ) self.logger.info("Running comparison tasks...") if config["fluxsite"]["multiprocess"]: @@ -308,10 +315,44 @@ def fluxsite(self, config_path: str, no_submit: bool, skip: list[str]): else: self.fluxsite_submit_job(config_path, skip) - def spatial(self, config_path: str): + def spatial_setup_work_directory(self, config_path: str): + """Endpoint for `benchcab spatial-setup-work-dir`.""" + config = self._get_config(config_path) + self._validate_environment(project=config["project"], modules=config["modules"]) + + self.logger.info("Setting up run directory tree for spatial tests...") + setup_spatial_directory_tree() + self.logger.info("Setting up tasks...") + try: + payu_config = config["spatial"]["payu"]["config"] + except KeyError: + payu_config = None + for task in self._get_spatial_tasks(config): + task.setup_task(payu_config=payu_config) + self.logger.info("Successfully setup spatial tasks") + + def spatial_run_tasks(self, config_path: str): + """Endpoint for `benchcab spatial-run-tasks`.""" + config = self._get_config(config_path) + self._validate_environment(project=config["project"], modules=config["modules"]) + + self.logger.info("Running spatial tasks...") + spatial.run_tasks(tasks=self._get_spatial_tasks(config)) + self.logger.info("Successfully dispatched payu jobs") + + def spatial(self, config_path: str, skip: list): """Endpoint for `benchcab spatial`.""" + self.checkout(config_path) + self.build(config_path, mpi=True) + self.spatial_setup_work_directory(config_path) + self.spatial_run_tasks(config_path) - def run(self, config_path: str, no_submit: bool, skip: list[str]): + def run(self, config_path: str, skip: list[str]): """Endpoint for `benchcab run`.""" - self.fluxsite(config_path, no_submit, skip) - self.spatial(config_path) + self.checkout(config_path) + self.build(config_path) + self.build(config_path, mpi=True) + self.fluxsite_setup_work_directory(config_path) + self.spatial_setup_work_directory(config_path) + self.fluxsite_submit_job(config_path, skip) + self.spatial_run_tasks(config_path) diff --git a/benchcab/cli.py b/benchcab/cli.py index 4e19d268..fee7086f 100644 --- a/benchcab/cli.py +++ b/benchcab/cli.py @@ -38,9 +38,9 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: action="store_true", ) - # parent parser that contains arguments common to all run specific subcommands - args_run_subcommand = argparse.ArgumentParser(add_help=False) - args_run_subcommand.add_argument( + # parent parser that contains the argument for --no-submit + args_no_submit = argparse.ArgumentParser(add_help=False) + args_no_submit.add_argument( "--no-submit", action="store_true", help="Force benchcab to execute tasks on the current compute node.", @@ -80,7 +80,6 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: parents=[ args_help, args_subcommand, - args_run_subcommand, args_composite_subcommand, ], help="Run all test suites for CABLE.", @@ -109,7 +108,7 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: parents=[ args_help, args_subcommand, - args_run_subcommand, + args_no_submit, args_composite_subcommand, ], help="Run the fluxsite test suite for CABLE.", @@ -140,6 +139,11 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: config file.""", add_help=False, ) + parser_build.add_argument( + "--mpi", + action="store_true", + help="Enable MPI build.", + ) parser_build.set_defaults(func=app.build) # subcommand: 'benchcab fluxsite-setup-work-dir' @@ -168,9 +172,9 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: "fluxsite-run-tasks", parents=[args_help, args_subcommand], help="Run the fluxsite tasks of the main fluxsite command.", - description="""Runs the fluxsite tasks for the fluxsite test suite. Note, this command should - ideally be run inside a PBS job. This command is invoked by the PBS job script generated by - `benchcab run`.""", + description="""Runs the fluxsite tasks for the fluxsite test suite. + Note, this command should ideally be run inside a PBS job. This command + is invoked by the PBS job script generated by `benchcab run`.""", add_help=False, ) parser_fluxsite_run_tasks.set_defaults(func=app.fluxsite_run_tasks) @@ -192,11 +196,32 @@ def generate_parser(app: Benchcab) -> argparse.ArgumentParser: # subcommand: 'benchcab spatial' parser_spatial = subparsers.add_parser( "spatial", - parents=[args_help, args_subcommand], + parents=[args_help, args_subcommand, args_composite_subcommand], help="Run the spatial tests only.", description="""Runs the default spatial test suite for CABLE.""", add_help=False, ) parser_spatial.set_defaults(func=app.spatial) + # subcommand: 'benchcab spatial-setup-work-dir' + parser_spatial_setup_work_dir = subparsers.add_parser( + "spatial-setup-work-dir", + parents=[args_help, args_subcommand], + help="Run the work directory setup step of the spatial command.", + description="""Generates the spatial run directory tree in the current working + directory so that spatial tasks can be run.""", + add_help=False, + ) + parser_spatial_setup_work_dir.set_defaults(func=app.spatial_setup_work_directory) + + # subcommand 'benchcab spatial-run-tasks' + parser_spatial_run_tasks = subparsers.add_parser( + "spatial-run-tasks", + parents=[args_help, args_subcommand], + help="Run the spatial tasks of the main spatial command.", + description="Runs the spatial tasks for the spatial test suite.", + add_help=False, + ) + parser_spatial_run_tasks.set_defaults(func=app.spatial_run_tasks) + return main_parser diff --git a/benchcab/config.py b/benchcab/config.py index 2cc51153..a144450c 100644 --- a/benchcab/config.py +++ b/benchcab/config.py @@ -96,6 +96,18 @@ def read_optional_key(config: dict): "science_configurations", internal.DEFAULT_SCIENCE_CONFIGURATIONS ) + # Default values for spatial + config["spatial"] = config.get("spatial", {}) + + config["spatial"]["met_forcings"] = config["spatial"].get( + "met_forcings", internal.SPATIAL_DEFAULT_MET_FORCINGS + ) + + config["spatial"]["payu"] = config["spatial"].get("payu", {}) + config["spatial"]["payu"]["config"] = config["spatial"]["payu"].get("config", {}) + config["spatial"]["payu"]["args"] = config["spatial"]["payu"].get("args") + + # Default values for fluxsite config["fluxsite"] = config.get("fluxsite", {}) config["fluxsite"]["multiprocess"] = config["fluxsite"].get( diff --git a/benchcab/data/config-schema.yml b/benchcab/data/config-schema.yml index 280c5601..4c9b560a 100644 --- a/benchcab/data/config-schema.yml +++ b/benchcab/data/config-schema.yml @@ -97,4 +97,27 @@ fluxsite: schema: type: "string" required: false - \ No newline at end of file + +spatial: + type: "dict" + required: false + schema: + met_forcings: + type: "dict" + required: false + minlength: 1 + keysrules: + type: "string" + valuesrules: + type: "string" + payu: + type: "dict" + required: false + schema: + config: + type: "dict" + required: false + args: + nullable: true + type: "string" + required: false \ No newline at end of file diff --git a/benchcab/data/test/config-optional.yml b/benchcab/data/test/config-optional.yml index 79b83a42..6ba4c332 100644 --- a/benchcab/data/test/config-optional.yml +++ b/benchcab/data/test/config-optional.yml @@ -11,6 +11,14 @@ fluxsite: storage: - scratch/$PROJECT +spatial: + met_forcings: + crujra_access: https://github.com/CABLE-LSM/cable_example.git + payu: + config: + walltime: "1:00:00" + args: -n 2 + science_configurations: - cable: cable_user: diff --git a/benchcab/environment_modules.py b/benchcab/environment_modules.py index fe845e80..0d26503c 100644 --- a/benchcab/environment_modules.py +++ b/benchcab/environment_modules.py @@ -14,7 +14,7 @@ try: from python import module except ImportError: - print( + get_logger().error( "Environment modules error: unable to import " "initialization script for python." ) diff --git a/benchcab/fluxsite.py b/benchcab/fluxsite.py index 1bf62764..816f5a46 100644 --- a/benchcab/fluxsite.py +++ b/benchcab/fluxsite.py @@ -7,9 +7,7 @@ import operator import shutil import sys -from pathlib import Path from subprocess import CalledProcessError -from typing import Any, Dict, TypeVar import f90nml import flatdict @@ -20,85 +18,9 @@ from benchcab.model import Model from benchcab.utils import get_logger from benchcab.utils.fs import chdir, mkdir +from benchcab.utils.namelist import patch_namelist, patch_remove_namelist from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface -# fmt: off -# ====================================================== -# Copyright (c) 2017 - 2022 Samuel Colvin and other contributors -# from https://github.com/pydantic/pydantic/blob/fd2991fe6a73819b48c906e3c3274e8e47d0f761/pydantic/utils.py#L200 - -KeyType = TypeVar('KeyType') - - -def deep_update(mapping: Dict[KeyType, Any], *updating_mappings: Dict[KeyType, Any]) -> Dict[KeyType, Any]: - """Perform a deep update of a mapping. - - Parameters - ---------- - mapping : Dict[KeyType, Any] - Mapping. - *updating_mappings : Dict[KeyType, Any] - Mapping updates. - - Returns - ------- - Dict[KeyType, Any] - Updated mapping. - - """ - updated_mapping = mapping.copy() - for updating_mapping in updating_mappings: - for k, v in updating_mapping.items(): - if k in updated_mapping and isinstance(updated_mapping[k], dict) and isinstance(v, dict): - updated_mapping[k] = deep_update(updated_mapping[k], v) - else: - updated_mapping[k] = v - return updated_mapping - -# ====================================================== -# fmt: on - - -def deep_del( - mapping: Dict[KeyType, Any], *updating_mappings: Dict[KeyType, Any] -) -> Dict[KeyType, Any]: - """Deletes all key-value 'leaf nodes' in `mapping` specified by `updating_mappings`.""" - updated_mapping = mapping.copy() - for updating_mapping in updating_mappings: - for key, value in updating_mapping.items(): - if isinstance(updated_mapping[key], dict) and isinstance(value, dict): - updated_mapping[key] = deep_del(updated_mapping[key], value) - else: - del updated_mapping[key] - return updated_mapping - - -def patch_namelist(nml_path: Path, patch: dict): - """Writes a namelist patch specified by `patch` to `nml_path`. - - The `patch` dictionary must comply with the `f90nml` api. - """ - if not nml_path.exists(): - f90nml.write(patch, nml_path) - return - - nml = f90nml.read(nml_path) - f90nml.write(deep_update(nml, patch), nml_path, force=True) - - -def patch_remove_namelist(nml_path: Path, patch_remove: dict): - """Removes a subset of namelist parameters specified by `patch_remove` from `nml_path`. - - The `patch_remove` dictionary must comply with the `f90nml` api. - """ - nml = f90nml.read(nml_path) - try: - f90nml.write(deep_del(nml, patch_remove), nml_path, force=True) - except KeyError as exc: - msg = f"Namelist parameters specified in `patch_remove` do not exist in {nml_path.name}." - raise KeyError(msg) from exc - - f90_logical_repr = {True: ".true.", False: ".false."} @@ -106,7 +28,7 @@ class CableError(Exception): """Custom exception class for CABLE errors.""" -class Task: +class FluxsiteTask: """A class used to represent a single fluxsite task.""" subprocess_handler: SubprocessWrapperInterface = SubprocessWrapper() @@ -353,10 +275,10 @@ def get_fluxsite_tasks( models: list[Model], science_configurations: list[dict], fluxsite_forcing_file_names: list[str], -) -> list[Task]: +) -> list[FluxsiteTask]: """Returns a list of fluxsite tasks to run.""" tasks = [ - Task( + FluxsiteTask( model=model, met_forcing_file=file_name, sci_conf_id=sci_conf_id, @@ -369,14 +291,15 @@ def get_fluxsite_tasks( return tasks -def run_tasks(tasks: list[Task]): +def run_tasks(tasks: list[FluxsiteTask]): """Runs tasks in `tasks` serially.""" for task in tasks: task.run() def run_tasks_in_parallel( - tasks: list[Task], n_processes=internal.FLUXSITE_DEFAULT_PBS["ncpus"] + tasks: list[FluxsiteTask], + n_processes=internal.FLUXSITE_DEFAULT_PBS["ncpus"], ): """Runs tasks in `tasks` in parallel across multiple processes.""" run_task = operator.methodcaller("run") @@ -384,7 +307,7 @@ def run_tasks_in_parallel( pool.map(run_task, tasks, chunksize=1) -def get_fluxsite_comparisons(tasks: list[Task]) -> list[ComparisonTask]: +def get_fluxsite_comparisons(tasks: list[FluxsiteTask]) -> list[ComparisonTask]: """Returns a list of `ComparisonTask` objects to run comparisons with. Pairs should be matching in science configurations and meteorological diff --git a/benchcab/internal.py b/benchcab/internal.py index 4f2ebc39..de7a0aaa 100644 --- a/benchcab/internal.py +++ b/benchcab/internal.py @@ -20,7 +20,6 @@ "walltime": "6:00:00", "storage": [], } -MPI = False FLUXSITE_DEFAULT_MULTIPROCESS = True # DIRECTORY PATHS/STRUCTURE: @@ -70,17 +69,43 @@ # Relative path to directory that stores bitwise comparison results FLUXSITE_DIRS["BITWISE_CMP"] = FLUXSITE_DIRS["ANALYSIS"] / "bitwise-comparisons" -# Path to met files: +# Relative path to root directory for CABLE spatial runs +SPATIAL_RUN_DIR = RUN_DIR / "spatial" + +# Relative path to tasks directory (contains payu control directories configured +# for each spatial task) +SPATIAL_TASKS_DIR = SPATIAL_RUN_DIR / "tasks" + +# A custom payu laboratory directory for payu runs +PAYU_LABORATORY_DIR = RUN_DIR / "payu-laboratory" + +# Path to PLUMBER2 site forcing data directory (doi: 10.25914/5fdb0902607e1): MET_DIR = Path("/g/data/ks32/CLEX_Data/PLUMBER2/v1-0/Met/") +# Default met forcings to use in the spatial test suite. Each met +# forcing has a corresponding payu experiment that is configured to run CABLE +# with that forcing. +SPATIAL_DEFAULT_MET_FORCINGS = { + "crujra_access": "https://github.com/CABLE-LSM/cable_example.git", +} + # CABLE SVN root url: CABLE_SVN_ROOT = "https://trac.nci.org.au/svn/cable" +# Relative path to temporary build directory (serial) +TMP_BUILD_DIR = Path("offline", ".tmp") + +# Relative path to temporary build directory (MPI) +TMP_BUILD_DIR_MPI = Path("offline", ".mpitmp") + # CABLE GitHub URL: CABLE_GIT_URL = "https://github.com/CABLE-LSM/CABLE.git" # CABLE executable file name: -CABLE_EXE = "cable-mpi" if MPI else "cable" +CABLE_EXE = "cable" + +# CABLE MPI executable file name: +CABLE_MPI_EXE = "cable-mpi" # CABLE namelist file name: CABLE_NML = "cable.nml" diff --git a/benchcab/model.py b/benchcab/model.py index 1bbf014b..0684de95 100644 --- a/benchcab/model.py +++ b/benchcab/model.py @@ -77,10 +77,14 @@ def model_id(self) -> int: def model_id(self, value: int): self._model_id = value - def get_exe_path(self) -> Path: + def get_exe_path(self, mpi=False) -> Path: """Return the path to the built executable.""" return ( - internal.SRC_DIR / self.name / self.src_dir / "offline" / internal.CABLE_EXE + internal.SRC_DIR + / self.name + / self.src_dir + / "offline" + / (internal.CABLE_MPI_EXE if mpi else internal.CABLE_EXE) ) def custom_build(self, modules: list[str]): @@ -112,10 +116,14 @@ def custom_build(self, modules: list[str]): with chdir(build_script_path.parent), self.modules_handler.load(modules): self.subprocess_handler.run_cmd(f"./{tmp_script_path.name}") - def pre_build(self): + def pre_build(self, mpi=False): """Runs CABLE pre-build steps.""" path_to_repo = internal.SRC_DIR / self.name - tmp_dir = path_to_repo / self.src_dir / "offline" / ".tmp" + tmp_dir = ( + path_to_repo + / self.src_dir + / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR) + ) if not tmp_dir.exists(): self.logger.debug(f"mkdir {tmp_dir}") tmp_dir.mkdir() @@ -128,10 +136,14 @@ def pre_build(self): copy2(path_to_repo / self.src_dir / "offline" / "Makefile", tmp_dir) - def run_build(self, modules: list[str]): + def run_build(self, modules: list[str], mpi=False): """Runs CABLE build scripts.""" path_to_repo = internal.SRC_DIR / self.name - tmp_dir = path_to_repo / self.src_dir / "offline" / ".tmp" + tmp_dir = ( + path_to_repo + / self.src_dir + / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR) + ) with chdir(tmp_dir), self.modules_handler.load(modules): env = os.environ.copy() @@ -140,20 +152,23 @@ def run_build(self, modules: list[str]): env["CFLAGS"] = "-O2 -fp-model precise" env["LDFLAGS"] = f"-L{env['NETCDF_ROOT']}/lib/Intel -O0" env["LD"] = "-lnetcdf -lnetcdff" - env["FC"] = "mpif90" if internal.MPI else "ifort" + env["FC"] = "mpif90" if mpi else "ifort" - self.subprocess_handler.run_cmd( - "make mpi" if internal.MPI else "make", env=env - ) + self.subprocess_handler.run_cmd("make mpi" if mpi else "make", env=env) - def post_build(self): + def post_build(self, mpi=False): """Runs CABLE post-build steps.""" path_to_repo = internal.SRC_DIR / self.name - tmp_dir = path_to_repo / self.src_dir / "offline" / ".tmp" + tmp_dir = ( + path_to_repo + / self.src_dir + / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR) + ) + exe = internal.CABLE_MPI_EXE if mpi else internal.CABLE_EXE rename( - tmp_dir / internal.CABLE_EXE, - path_to_repo / self.src_dir / "offline" / internal.CABLE_EXE, + tmp_dir / exe, + path_to_repo / self.src_dir / "offline" / exe, ) diff --git a/benchcab/spatial.py b/benchcab/spatial.py new file mode 100644 index 00000000..2d1d115a --- /dev/null +++ b/benchcab/spatial.py @@ -0,0 +1,145 @@ +# Copyright 2022 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. +# SPDX-License-Identifier: Apache-2.0 + +"""A module containing functions and data structures for running spatial tasks.""" + +from typing import Optional + +import git +import yaml + +from benchcab import internal +from benchcab.model import Model +from benchcab.utils import get_logger +from benchcab.utils.dict import deep_update +from benchcab.utils.fs import chdir +from benchcab.utils.namelist import patch_namelist, patch_remove_namelist +from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface + + +class SpatialTask: + """A class used to represent a single spatial task.""" + + subprocess_handler: SubprocessWrapperInterface = SubprocessWrapper() + + def __init__( + self, + model: Model, + met_forcing_name: str, + met_forcing_payu_experiment: str, + sci_conf_id: int, + sci_config: dict, + payu_args: Optional[str] = None, + ) -> None: + self.model = model + self.met_forcing_name = met_forcing_name + self.met_forcing_payu_experiment = met_forcing_payu_experiment + self.sci_conf_id = sci_conf_id + self.sci_config = sci_config + self.payu_args = payu_args + self.logger = get_logger() + + def get_task_name(self) -> str: + """Returns the file name convention used for this task.""" + return f"{self.met_forcing_name}_R{self.model.model_id}_S{self.sci_conf_id}" + + def setup_task(self, payu_config: Optional[dict] = None): + """Does all file manipulations to run cable with payu for this task.""" + self.logger.debug(f"Setting up task: {self.get_task_name()}") + + self.clone_experiment() + self.configure_experiment(payu_config) + self.update_namelist() + + def clone_experiment(self): + """Clone the payu experiment from GitHub.""" + url = self.met_forcing_payu_experiment + path = internal.SPATIAL_TASKS_DIR / self.get_task_name() + self.logger.debug(f"git clone {url} {path}") + _ = git.Repo.clone_from(url, path) + + def configure_experiment(self, payu_config: Optional[dict] = None): + """Configure the payu experiment for this task.""" + task_dir = internal.SPATIAL_TASKS_DIR / self.get_task_name() + exp_config_path = task_dir / "config.yaml" + with exp_config_path.open("r", encoding="utf-8") as file: + config = yaml.safe_load(file) + if config is None: + config = {} + + self.logger.debug( + f" Updating experiment config parameters in {task_dir / 'config.yaml'}" "" + ) + + if payu_config: + config = deep_update(config, payu_config) + + config["exe"] = str(self.model.get_exe_path(mpi=True).absolute()) + + # Here we prepend inputs to the `input` list so that payu knows to use + # our inputs over the pre-existing inputs in the config file: + config["input"] = config.get("input", []) + + config["laboratory"] = str(internal.PAYU_LABORATORY_DIR.absolute()) + + with exp_config_path.open("w", encoding="utf-8") as file: + yaml.dump(config, file) + + def update_namelist(self): + """Update the namelist file for this task.""" + nml_path = ( + internal.SPATIAL_TASKS_DIR / self.get_task_name() / internal.CABLE_NML + ) + self.logger.debug( + f" Adding science configurations to CABLE namelist file {nml_path}" + ) + patch_namelist(nml_path, self.sci_config) + + if self.model.patch: + self.logger.debug( + f" Adding branch specific configurations to CABLE namelist file {nml_path}" + ) + patch_namelist(nml_path, self.model.patch) + + if self.model.patch_remove: + self.logger.debug( + f" Removing branch specific configurations from CABLE namelist file {nml_path}" + ) + patch_remove_namelist(nml_path, self.model.patch_remove) + + def run(self) -> None: + """Runs a single spatial task.""" + task_dir = internal.SPATIAL_TASKS_DIR / self.get_task_name() + with chdir(task_dir): + self.subprocess_handler.run_cmd( + f"payu run {self.payu_args}" if self.payu_args else "payu run", + ) + + +def run_tasks(tasks: list[SpatialTask]): + """Runs tasks in `tasks` sequentially.""" + for task in tasks: + task.run() + + +def get_spatial_tasks( + models: list[Model], + met_forcings: dict[str, str], + science_configurations: list[dict], + payu_args: Optional[str] = None, +): + """Returns a list of spatial tasks to run.""" + tasks = [ + SpatialTask( + model=model, + met_forcing_name=met_forcing_name, + met_forcing_payu_experiment=met_forcing_payu_experiment, + sci_conf_id=sci_conf_id, + sci_config=sci_config, + payu_args=payu_args, + ) + for model in models + for met_forcing_name, met_forcing_payu_experiment in met_forcings.items() + for sci_conf_id, sci_config in enumerate(science_configurations) + ] + return tasks diff --git a/benchcab/utils/dict.py b/benchcab/utils/dict.py new file mode 100644 index 00000000..c76e6c78 --- /dev/null +++ b/benchcab/utils/dict.py @@ -0,0 +1,41 @@ +# Copyright 2022 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. +# SPDX-License-Identifier: Apache-2.0 + +"""Utility functions for manipulating nested dictionaries.""" + +from typing import Any, Dict, TypeVar + +# fmt: off +# ====================================================== +# Copyright (c) 2017 - 2022 Samuel Colvin and other contributors +# from https://github.com/pydantic/pydantic/blob/fd2991fe6a73819b48c906e3c3274e8e47d0f761/pydantic/utils.py#L200 + +KeyType = TypeVar('KeyType') + + +def deep_update(mapping: Dict[KeyType, Any], *updating_mappings: Dict[KeyType, Any]) -> Dict[KeyType, Any]: # noqa + updated_mapping = mapping.copy() + for updating_mapping in updating_mappings: + for k, v in updating_mapping.items(): + if k in updated_mapping and isinstance(updated_mapping[k], dict) and isinstance(v, dict): + updated_mapping[k] = deep_update(updated_mapping[k], v) + else: + updated_mapping[k] = v + return updated_mapping + +# ====================================================== +# fmt: on + + +def deep_del( + mapping: Dict[KeyType, Any], *updating_mappings: Dict[KeyType, Any] +) -> Dict[KeyType, Any]: + """Deletes all key-value 'leaf nodes' in `mapping` specified by `updating_mappings`.""" + updated_mapping = mapping.copy() + for updating_mapping in updating_mappings: + for key, value in updating_mapping.items(): + if isinstance(updated_mapping[key], dict) and isinstance(value, dict): + updated_mapping[key] = deep_del(updated_mapping[key], value) + else: + del updated_mapping[key] + return updated_mapping diff --git a/benchcab/utils/namelist.py b/benchcab/utils/namelist.py new file mode 100644 index 00000000..d6a6de48 --- /dev/null +++ b/benchcab/utils/namelist.py @@ -0,0 +1,36 @@ +# Copyright 2022 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. +# SPDX-License-Identifier: Apache-2.0 + +"""Contains utility functions for manipulating Fortran namelist files.""" + +from pathlib import Path + +import f90nml + +from benchcab.utils.dict import deep_del, deep_update + + +def patch_namelist(nml_path: Path, patch: dict): + """Writes a namelist patch specified by `patch` to `nml_path`. + + The `patch` dictionary must comply with the `f90nml` api. + """ + if not nml_path.exists(): + f90nml.write(patch, nml_path) + return + + nml = f90nml.read(nml_path) + f90nml.write(deep_update(nml, patch), nml_path, force=True) + + +def patch_remove_namelist(nml_path: Path, patch_remove: dict): + """Removes a subset of namelist parameters specified by `patch_remove` from `nml_path`. + + The `patch_remove` dictionary must comply with the `f90nml` api. + """ + nml = f90nml.read(nml_path) + try: + f90nml.write(deep_del(nml, patch_remove), nml_path, force=True) + except KeyError as exc: + msg = f"Namelist parameters specified in `patch_remove` do not exist in {nml_path.name}." + raise KeyError(msg) from exc diff --git a/benchcab/workdir.py b/benchcab/workdir.py index 27a0a1ca..d5a8e40f 100644 --- a/benchcab/workdir.py +++ b/benchcab/workdir.py @@ -22,3 +22,13 @@ def setup_fluxsite_directory_tree(): """Generate the directory structure used by `benchcab`.""" for path in internal.FLUXSITE_DIRS.values(): mkdir(path, parents=True, exist_ok=True) + + +def setup_spatial_directory_tree(): + """Generate the directory structure for running spatial tests.""" + for path in [ + internal.SPATIAL_RUN_DIR, + internal.SPATIAL_TASKS_DIR, + internal.PAYU_LABORATORY_DIR, + ]: + mkdir(path, parents=True, exist_ok=True) diff --git a/docs/user_guide/config_options.md b/docs/user_guide/config_options.md index 2026a725..9da6b5dd 100644 --- a/docs/user_guide/config_options.md +++ b/docs/user_guide/config_options.md @@ -163,6 +163,69 @@ fluxsites: ``` +## spatial + +Contains settings specific to spatial tests. + +This key is _optional_. **Default** settings for the spatial tests will be used if it is not present. + +```yaml +spatial: + met_forcings: + crujra_access: https://github.com/CABLE-LSM/cable_example.git + payu: + config: + walltime: 1:00:00 + args: -n 2 +``` + +### [met_forcings](#met_forcings) + +Specify one or more spatial met forcings to use in the spatial test suite. Each entry is a key-value pair where the key is the name of the met forcing and the value is a URL to a payu experiment that is configured to run CABLE with that forcing. + +This key is _optional_. **Default** values for the `met_forcings` key is as follows: + +```yaml +spatial: + met_forcings: + crujra_access: https://github.com/CABLE-LSM/cable_example.git +``` + +### [payu](#payu) + +Contains settings specific to the payu workflow manager. + +This key is _optional_. **Default** values for the payu settings will apply if not specified. + +```yaml +spatial: + payu: + config: + walltime: 1:00:00 + args: -n 2 +``` + +[`config`](#+payu.config){ #+payu.config } + +: **Default:** unset, _optional key_. :octicons-dash-24: Specify global configuration options for running payu. Settings specified here are passed into to the payu configuration file for each experiment. + +```yaml +spatial: + payu: + config: + walltime: 1:00:00 +``` + +[`args`](#+payu.args){ #+payu.args } + +: **Default:** unset, _optional key_. :octicons-dash-24: Specify command line arguments to the `payu run` command in the form of a string. Arguments are used for all spatial payu runs. + +```yaml +spatial: + payu: + args: -n 2 +``` + ## realisations Entries for each CABLE branch to use. Each entry is a key-value pair and are listed as follows: @@ -337,7 +400,7 @@ realisations: ### [patch_remove](#patch_remove) -: **Default:** unset, no effect, _optional key. :octicons-dash-24: Specifies branch-specific namelist settings to be removed from the `cable.nml` namelist settings. When the `patch_remove` key is specified, the specified namelists are removed from all namelist files for this branch for all science configurations run by `benchcab`. When specifying a namelist parameter in `patch_remove`, the value of the namelist parameter is ignored. +: **Default:** unset, _optional key. :octicons-dash-24: Specifies branch-specific namelist settings to be removed from the `cable.nml` namelist settings. When the `patch_remove` key is specified, the specified namelists are removed from all namelist files for this branch for all science configurations run by `benchcab`. When specifying a namelist parameter in `patch_remove`, the value of the namelist parameter is ignored. : The `patch_remove` key must be a dictionary-like data structure that is compliant with the [`f90nml`][f90nml-github] python package. ```yaml @@ -356,7 +419,7 @@ realisations: ## science_configurations -: **Default:** unset, no impact, _optional key_. :octicons-dash-24: User defined science configurations. Science configurations that are specified here will replace [the default science configurations](default_science_configurations.md). In the output filenames, each configuration is identified with S where N is an integer starting from 0 for the first listed configuration and increasing by 1 for each subsequent configuration. +: **Default:** unset, _optional key_. :octicons-dash-24: User defined science configurations. Science configurations that are specified here will replace [the default science configurations](default_science_configurations.md). In the output filenames, each configuration is identified with S where N is an integer starting from 0 for the first listed configuration and increasing by 1 for each subsequent configuration. ```yaml science_configurations: [ diff --git a/docs/user_guide/expected_output.md b/docs/user_guide/expected_output.md index a126fb69..66963a4d 100644 --- a/docs/user_guide/expected_output.md +++ b/docs/user_guide/expected_output.md @@ -6,11 +6,11 @@ Other sub-commands should print out part of this output. ``` $ benchcab run -Creating src directory: /scratch/tm70/sb8430/bench_example/src +Creating src directory Checking out repositories... -Successfully checked out trunk at revision 9550 -Successfully checked out test-branch at revision 9550 -Successfully checked out CABLE-AUX at revision 9550 +Successfully checked out trunk at revision 9672 +Successfully checked out test-branch at revision 9672 +Successfully checked out CABLE-AUX at revision 9672 Writing revision number info to rev_number-1.log Compiling CABLE serially for realisation trunk... @@ -18,24 +18,31 @@ Successfully compiled CABLE for realisation trunk Compiling CABLE serially for realisation test-branch... Successfully compiled CABLE for realisation test-branch +Compiling CABLE with MPI for realisation trunk... +Successfully compiled CABLE for realisation trunk +Compiling CABLE with MPI for realisation test-branch... +Successfully compiled CABLE for realisation test-branch + Setting up run directory tree for fluxsite tests... -Creating runs/fluxsite/logs directory: /scratch/tm70/sb8430/bench_example/runs/fluxsite/logs -Creating runs/fluxsite/outputs directory: /scratch/tm70/sb8430/bench_example/runs/fluxsite/outputs -Creating runs/fluxsite/tasks directory: /scratch/tm70/sb8430/bench_example/runs/fluxsite/tasks -Creating runs/fluxsite/analysis directory: /scratch/tm70/sb8430/bench_example/runs/fluxsite/analysis -Creating runs/fluxsite/analysis/bitwise-comparisons directory: /scratch/tm70/sb8430/bench_example/runs/fluxsite/analysis/bitwise-comparisons -Creating task directories... Setting up tasks... Successfully setup fluxsite tasks +Setting up run directory tree for spatial tests... +Setting up tasks... +Successfully setup spatial tasks + Creating PBS job script to run fluxsite tasks on compute nodes: benchmark_cable_qsub.sh -PBS job submitted: 82479088.gadi-pbs +PBS job submitted: 100563227.gadi-pbs The CABLE log file for each task is written to runs/fluxsite/logs/_log.txt The CABLE standard output for each task is written to runs/fluxsite/tasks//out.txt The NetCDF output for each task is written to runs/fluxsite/outputs/_out.nc + +Running spatial tasks... +Successfully dispatched payu jobs + ``` -The PBS schedule job should print out the following to the job log file: +The benchmark_cable_qsub.sh PBS job should print out the following to the job log file: ``` Running fluxsite tasks... Successfully ran fluxsite tasks diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index ead57da0..23c836b6 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -8,12 +8,6 @@ In this guide, we will describe: `benchcab` has been designed to work on NCI machine exclusively. It might be extended later on to other systems. -!!! warning "Limitations" - Currently, - - * `benchcab` can only run simulations at flux sites. - * spin-up for CASA simulations are not supported. - ## Pre-requisites To use `benchcab`, you need to join the following projects at NCI: @@ -45,15 +39,15 @@ You need to load the module on each new session at NCI on login or compute nodes - *New feature:* running two versions of CABLE with the same standard set of science configurations except one version is patched to use a new feature. - *Ensemble run:* running any number of versions of CABLE with the same set of customised science configurations. -The regression and new feature run modes should be used as necessary when evaluating new development in CABLE. - -The code will: (i) check out and (ii) build the code branches. Then it will run each executable across N standard science configurations for a given number of sites. It is possible to produce some plots locally from the output produced. But [the modelevaluation website][meorg] can be used for further benchmarking and evaluation. +The regression and new feature run modes should be used as necessary when evaluating new developments in CABLE. ### Create a work directory #### Choose a location -You can run the benchmark from any directory you want under `/scratch` or `/g/data`. `/scratch` is preferred as the data in the run directory does not need to be preserved for a long time. The code will create sub-directories as needed. Please ensure you have enough space to store the CABLE outputs in your directory, at least temporary, until you upload them to [modelevaluation.org][meorg]. You will need about 16GB for the outputs for the `forty-two-site` experiment (with 4 different science configurations). +You can run the benchmark from any directory you want under `/scratch` or `/g/data`. `/scratch` is preferred as the data in the run directory does not need to be preserved for a long time. The code will create sub-directories as needed. Please ensure you have enough space to store the CABLE outputs in your directory, at least temporarily until you upload them to [modelevaluation.org][meorg]. + +The full test suite will require about 22GB of storage space. !!! Warning "The HOME directory is unsuitable" @@ -84,7 +78,7 @@ cd bench_example !!! warning `benchcab` will stop if it is not run within a work directory with the proper structure. -Currently, `benchcab` can only run CABLE for flux sites. **To run the whole workflow**, run +Currently, `benchcab` can only run CABLE for flux site and offline spatial configurations. **To run the whole workflow**, run ```bash benchcab run @@ -94,7 +88,8 @@ The tool will follow the steps: 1. Checkout the code branches. The codes will be stored under `src/` directory in your work directory. The sub-directories are created automatically. 2. Compile the source code from all branches -3. Setup and launch a PBS job to run the simulations in parallel. When `benchcab` launches the PBS job, it will print out the job ID to the terminal. You can check the status of the job with `qstat`. `benchcab` will not warn you when the simulations are over. +3. Setup and launch a PBS job to run the flux site simulations in parallel. When `benchcab` launches the PBS job, it will print out the job ID to the terminal. You can check the status of the job with `qstat`. `benchcab` will not warn you when the simulations are over. +4. Setup and run an ensemble of offline spatial runs using the [`payu`][payu-github] framework. !!! tip "Expected output" @@ -120,22 +115,27 @@ The following files and directories are created when `benchcab run` executes suc ├── benchmark_cable_qsub.sh.o ├── rev_number-1.log ├── runs -│   └── fluxsite -│   ├── logs -│ │ ├── _log.txt -│ │ └── ... -│   ├── outputs -│ │ ├── _out.nc -│ │ └── ... -│   ├── analysis -│ │ └── bitwise-comparisons -│   └── tasks -│ ├── -│ │ ├── cable (executable) -│ │ ├── cable.nml -│ │ ├── cable_soilparm.nml -│ │ └── pft_params.nml -│ └── ... +│   ├── fluxsite +│   │ ├── logs +│ │ │ ├── _log.txt +│ │ │ └── ... +│   │ ├── outputs +│ │ │ ├── _out.nc +│ │ │ └── ... +│   │ ├── analysis +│ │ │ └── bitwise-comparisons +│   │ └── tasks +│ │ ├── +│ │ │ ├── cable (executable) +│ │ │ ├── cable.nml +│ │ │ ├── cable_soilparm.nml +│ │ │ └── pft_params.nml +│ │ └── ... +│   ├── spatial +│   │ └── tasks +│ │ ├── (a payu control / experiment directory) +│ │ └── ... +│   └── payu-laboratory └── src ├── CABLE-AUX ├── @@ -156,11 +156,11 @@ The following files and directories are created when `benchcab run` executes suc `runs/fluxsite/` -: directory that contains the log files, output files, and tasks for running CABLE. +: directory that contains the log files, output files, and tasks for running CABLE in the fluxsite configuration. `runs/fluxsite/tasks` -: directory that contains task directories. A task consists of a CABLE run for a branch (realisation), a meteorological forcing, and a science configuration. In the above directory structure, `` uses the following naming convention: +: directory that contains fluxsite task directories. A task consists of a CABLE run for a branch (realisation), a meteorological forcing, and a science configuration. In the above directory structure, `` uses the following naming convention: ``` _R_S @@ -184,6 +184,29 @@ The following files and directories are created when `benchcab run` executes suc : directory that contains the standard output produced by the bitwise comparison command: `benchcab fluxsite-bitwise-cmp`. Standard output is only saved when the netcdf files being compared differ from each other +`runs/spatial/` + +: directory that contains task directories for running CABLE in the offline spatial configuration. + +`runs/spatial/tasks` + +: directory that contains payu control directories (or experiments) configured for each spatial task. A task consists of a CABLE run for a branch (realisation), a meteorological forcing, and a science configuration. In the above directory structure, `` uses the following naming convention: + +``` +_R_S +``` + +: where `met_forcing_name` is the name of the spatial met forcing, `realisation_key` is the branch key specified in the config file, and `science_config_key` identifies the science configuration used. See the [`met_forcings`](config_options.md#met_forcings) option for more information on how to configure the met forcings used. + + +`runs/spatial/tasks//` + +: a payu control directory (or experiment). See [Configuring your experiment](https://payu.readthedocs.io/en/latest/config.html) for more information on payu experiments. + +`runs/payu-laboratory/` + +: a custom payu laboratory directory. See [Laboratory Structure](https://payu.readthedocs.io/en/latest/design.html#laboratory-structure) for more information on the payu laboratory directory. + !!! warning "Re-running `benchcab` multiple times in the same working directory" We recommend the user to manually delete the generated files when re-running `benchcab`. Re-running `benchcab` multiple times in the same working directory is currently not yet supported (see issue [CABLE-LSM/benchcab#20](https://github.com/CABLE-LSM/benchcab/issues/20)). To clean the current working directory, run the following command in the working directory @@ -193,10 +216,11 @@ The following files and directories are created when `benchcab run` executes suc ## Analyse the output with [modelevaluation.org][meorg] - - Once the benchmarking has finished running all the simulations, you need to upload the output files to [modelevaluation.org][meorg] via the web interface. To do this: +!!! warning "Limitations" + Model evaluation for offline spatial outputs is not yet available (see issue [CABLE-LSM/benchcab#193](https://github.com/CABLE-LSM/benchcab/issues/193)). + 1. Go to [modelevaluation.org][meorg] and login or create a new account. 2. Navigate to the `benchcab-evaluation` workspace. To do this, click the **Current Workspace** button at the top of the page, and select `benchcab-evaluation` under "Workspaces Shared With Me".
@@ -275,3 +299,4 @@ Alternatively, you can also access the ACCESS-NRI User support via [the ACCESS-H [benchmark_5]: https://modelevaluation.org/modelOutput/display/diLdf49PfpEwZemTz [benchmark_42]: https://modelevaluation.org/modelOutput/display/pvkuY5gpR2n4FKZw3 [run_CABLE_v2]: running_CABLE_v2.md +[payu-github]: https://github.com/payu-org/payu diff --git a/tests/conftest.py b/tests/conftest.py index b2a6116e..39c807a3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -90,6 +90,20 @@ def config(): }, "multiprocess": True, }, + "spatial": { + "met_forcings": { + "crujra_access": "https://github.com/CABLE-LSM/cable_example.git", + "gswp": "foo", + }, + "payu": { + "config": { + "ncpus": 16, + "walltime": "1:00:00", + "mem": "64GB", + }, + "args": "-n 2", + }, + }, } diff --git a/tests/test_cli.py b/tests/test_cli.py index f4c4e038..c190d25c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,7 +15,6 @@ def test_cli_parser(): res = vars(parser.parse_args(["run"])) assert res == { "config_path": "config.yaml", - "no_submit": False, "verbose": False, "skip": [], "func": app.run, @@ -42,6 +41,7 @@ def test_cli_parser(): assert res == { "config_path": "config.yaml", "verbose": False, + "mpi": False, "func": app.build, } @@ -93,9 +93,26 @@ def test_cli_parser(): assert res == { "config_path": "config.yaml", "verbose": False, + "skip": [], "func": app.spatial, } + # Success case: default spatial-setup-work-dir command + res = vars(parser.parse_args(["spatial-setup-work-dir"])) + assert res == { + "config_path": "config.yaml", + "verbose": False, + "func": app.spatial_setup_work_directory, + } + + # Success case: default spatial-run-tasks command + res = vars(parser.parse_args(["spatial-run-tasks"])) + assert res == { + "config_path": "config.yaml", + "verbose": False, + "func": app.spatial_run_tasks, + } + # Failure case: pass --no-submit to a non 'run' command with pytest.raises(SystemExit): parser.parse_args(["fluxsite-setup-work-dir", "--no-submit"]) diff --git a/tests/test_config.py b/tests/test_config.py index dcf306b2..de4cb14e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -11,6 +11,7 @@ import benchcab.config as bc import benchcab.internal as bi import benchcab.utils as bu +from benchcab import internal NO_OPTIONAL_CONFIG_PROJECT = "hh5" OPTIONAL_CONFIG_PROJECT = "ks32" @@ -76,6 +77,10 @@ def all_optional_default_config(no_optional_config) -> dict: "pbs": bi.FLUXSITE_DEFAULT_PBS, }, "science_configurations": bi.DEFAULT_SCIENCE_CONFIGURATIONS, + "spatial": { + "payu": {"config": {}, "args": None}, + "met_forcings": internal.SPATIAL_DEFAULT_MET_FORCINGS, + }, } for c_r in config["realisations"]: c_r["name"] = None @@ -108,6 +113,12 @@ def all_optional_custom_config(no_optional_config) -> dict: } } ], + "spatial": { + "payu": {"config": {"walltime": "1:00:00"}, "args": "-n 2"}, + "met_forcings": { + "crujra_access": "https://github.com/CABLE-LSM/cable_example.git" + }, + }, } branch_names = ["svn_trunk", "git_branch"] diff --git a/tests/test_fluxsite.py b/tests/test_fluxsite.py index 36a9b63b..4a77ef82 100644 --- a/tests/test_fluxsite.py +++ b/tests/test_fluxsite.py @@ -6,7 +6,6 @@ """ import math -from pathlib import Path import f90nml import netCDF4 @@ -15,12 +14,10 @@ from benchcab import __version__, internal from benchcab.fluxsite import ( CableError, - Task, + FluxsiteTask, get_comparison_name, get_fluxsite_comparisons, get_fluxsite_tasks, - patch_namelist, - patch_remove_namelist, ) from benchcab.model import Model from benchcab.utils.repo import Repo @@ -59,8 +56,8 @@ def model(mock_subprocess_handler, mock_repo): @pytest.fixture() def task(model, mock_subprocess_handler): - """Returns a mock `Task` instance.""" - _task = Task( + """Returns a mock `FluxsiteTask` instance.""" + _task = FluxsiteTask( model=model, met_forcing_file="forcing-file.nc", sci_conf_id=0, @@ -71,7 +68,7 @@ def task(model, mock_subprocess_handler): class TestGetTaskName: - """tests for `Task.get_task_name()`.""" + """tests for `FluxsiteTask.get_task_name()`.""" def test_task_name_convention(self, task): """Success case: check task name convention.""" @@ -79,7 +76,7 @@ def test_task_name_convention(self, task): class TestGetLogFilename: - """Tests for `Task.get_log_filename()`.""" + """Tests for `FluxsiteTask.get_log_filename()`.""" def test_log_filename_convention(self, task): """Success case: check log file name convention.""" @@ -87,7 +84,7 @@ def test_log_filename_convention(self, task): class TestGetOutputFilename: - """Tests for `Task.get_output_filename()`.""" + """Tests for `FluxsiteTask.get_output_filename()`.""" def test_output_filename_convention(self, task): """Success case: check output file name convention.""" @@ -95,11 +92,11 @@ def test_output_filename_convention(self, task): class TestFetchFiles: - """Tests for `Task.fetch_files()`.""" + """Tests for `FluxsiteTask.fetch_files()`.""" @pytest.fixture(autouse=True) def _setup(self, task): - """Setup precondition for `Task.fetch_files()`.""" + """Setup precondition for `FluxsiteTask.fetch_files()`.""" internal.NAMELIST_DIR.mkdir() (internal.NAMELIST_DIR / internal.CABLE_NML).touch() (internal.NAMELIST_DIR / internal.CABLE_SOIL_NML).touch() @@ -125,11 +122,11 @@ def test_required_files_are_copied_to_task_dir(self, task): class TestCleanTask: - """Tests for `Task.clean_task()`.""" + """Tests for `FluxsiteTask.clean_task()`.""" @pytest.fixture(autouse=True) def _setup(self, task): - """Setup precondition for `Task.clean_task()`.""" + """Setup precondition for `FluxsiteTask.clean_task()`.""" task_dir = internal.FLUXSITE_DIRS["TASKS"] / task.get_task_name() task_dir.mkdir(parents=True) (task_dir / internal.CABLE_NML).touch() @@ -157,91 +154,12 @@ def test_clean_files(self, task): assert not (internal.FLUXSITE_DIRS["LOG"] / task.get_log_filename()).exists() -class TestPatchNamelist: - """Tests for `patch_namelist()`.""" - - @pytest.fixture() - def nml_path(self): - """Return a path to a namelist file used for testing.""" - return Path("test.nml") - - def test_patch_on_non_existing_namelist_file(self, nml_path): - """Success case: patch non-existing namelist file.""" - patch = {"cable": {"file": "/path/to/file", "bar": 123}} - patch_namelist(nml_path, patch) - assert f90nml.read(nml_path) == patch - - def test_patch_on_non_empty_namelist_file(self, nml_path): - """Success case: patch non-empty namelist file.""" - f90nml.write({"cable": {"file": "/path/to/file", "bar": 123}}, nml_path) - patch_namelist(nml_path, {"cable": {"some": {"parameter": True}, "bar": 456}}) - assert f90nml.read(nml_path) == { - "cable": { - "file": "/path/to/file", - "bar": 456, - "some": {"parameter": True}, - } - } - - def test_empty_patch_does_nothing(self, nml_path): - """Success case: empty patch does nothing.""" - f90nml.write({"cable": {"file": "/path/to/file", "bar": 123}}, nml_path) - prev = f90nml.read(nml_path) - patch_namelist(nml_path, {}) - assert f90nml.read(nml_path) == prev - - -class TestPatchRemoveNamelist: - """Tests for `patch_remove_namelist()`.""" - - @pytest.fixture() - def nml(self): - """Return a namelist dictionary used for testing.""" - return { - "cable": { - "cable_user": { - "some_parameter": True, - "new_feature": True, - }, - }, - } - - @pytest.fixture() - def nml_path(self, nml): - """Create a namelist file and return its path.""" - _nml_path = Path("test.nml") - f90nml.write(nml, _nml_path) - return _nml_path - - def test_remove_namelist_parameter_from_derived_type(self, nml_path): - """Success case: remove a namelist parameter from derrived type.""" - patch_remove_namelist( - nml_path, {"cable": {"cable_user": {"new_feature": True}}} - ) - assert f90nml.read(nml_path) == { - "cable": {"cable_user": {"some_parameter": True}} - } - - def test_empty_patch_remove_does_nothing(self, nml_path, nml): - """Success case: empty patch_remove does nothing.""" - patch_remove_namelist(nml_path, {}) - assert f90nml.read(nml_path) == nml - - def test_key_error_raised_for_non_existent_namelist_parameter(self, nml_path): - """Failure case: test patch_remove KeyError exeption.""" - with pytest.raises( - KeyError, - match=f"Namelist parameters specified in `patch_remove` do not exist in {nml_path.name}.", - ): - patch_remove_namelist(nml_path, {"cable": {"foo": {"bar": True}}}) - - class TestSetupTask: - """Tests for `Task.setup_task()`.""" + """Tests for `FluxsiteTask.setup_task()`.""" @pytest.fixture(autouse=True) def _setup(self, task): - """Setup precondition for `Task.setup_task()`.""" + """Setup precondition for `FluxsiteTask.setup_task()`.""" (internal.NAMELIST_DIR).mkdir() (internal.NAMELIST_DIR / internal.CABLE_NML).touch() (internal.NAMELIST_DIR / internal.CABLE_SOIL_NML).touch() @@ -284,11 +202,11 @@ def test_all_settings_are_patched_into_namelist_file(self, task): class TestRunCable: - """Tests for `Task.run_cable()`.""" + """Tests for `FluxsiteTask.run_cable()`.""" @pytest.fixture(autouse=True) def _setup(self, task): - """Setup precondition for `Task.run_cable()`.""" + """Setup precondition for `FluxsiteTask.run_cable()`.""" task_dir = internal.FLUXSITE_DIRS["TASKS"] / task.get_task_name() task_dir.mkdir(parents=True) @@ -310,7 +228,7 @@ def test_cable_error_exception(self, task, mock_subprocess_handler): class TestAddProvenanceInfo: - """Tests for `Task.add_provenance_info()`.""" + """Tests for `FluxsiteTask.add_provenance_info()`.""" @pytest.fixture() def nml(self): @@ -334,7 +252,7 @@ def nc_output_path(self, task): @pytest.fixture(autouse=True) def _setup(self, task, nml): - """Setup precondition for `Task.add_provenance_info()`.""" + """Setup precondition for `FluxsiteTask.add_provenance_info()`.""" task_dir = internal.FLUXSITE_DIRS["TASKS"] / task.get_task_name() task_dir.mkdir(parents=True) fluxsite_output_dir = internal.FLUXSITE_DIRS["OUTPUT"] @@ -361,7 +279,7 @@ class TestGetFluxsiteTasks: @pytest.fixture() def models(self, mock_repo): - """Return a list of `CableRepository` instances used for testing.""" + """Return a list of `Model` instances used for testing.""" return [Model(repo=mock_repo, model_id=id) for id in range(2)] @pytest.fixture() @@ -403,7 +321,7 @@ class TestGetFluxsiteComparisons: def test_comparisons_for_two_branches_with_two_tasks(self, mock_repo): """Success case: comparisons for two branches with two tasks.""" tasks = [ - Task( + FluxsiteTask( model=Model(repo=mock_repo, model_id=model_id), met_forcing_file="foo.nc", sci_config={"foo": "bar"}, @@ -426,7 +344,7 @@ def test_comparisons_for_two_branches_with_two_tasks(self, mock_repo): def test_comparisons_for_three_branches_with_three_tasks(self, mock_repo): """Success case: comparisons for three branches with three tasks.""" tasks = [ - Task( + FluxsiteTask( model=Model(repo=mock_repo, model_id=model_id), met_forcing_file="foo.nc", sci_config={"foo": "bar"}, diff --git a/tests/test_model.py b/tests/test_model.py index 5ae06dfa..ece9b096 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -25,18 +25,24 @@ class MockRepo(Repo): def __init__(self) -> None: self.handle = "trunk" - def checkout(self): + def checkout(self, path: Path): pass def get_branch_name(self) -> str: return self.handle - def get_revision(self) -> str: + def get_revision(self, path: Path) -> str: pass return MockRepo() +@pytest.fixture(params=[False, True]) +def mpi(request): + """Return a parametrized mpi flag for testing.""" + return request.param + + @pytest.fixture() def model(mock_repo, mock_subprocess_handler, mock_environment_modules_handler): """Return a mock `Model` instance for testing against.""" @@ -67,11 +73,15 @@ def test_undefined_model_id(self, model): class TestGetExePath: """Tests for `Model.get_exe_path()`.""" - def test_serial_exe_path(self, model): - """Success case: get path to serial executable.""" + @pytest.mark.parametrize( + ("mpi", "expected_exe"), + [(False, internal.CABLE_EXE), (True, internal.CABLE_MPI_EXE)], + ) + def test_get_exe_path(self, model, mpi, expected_exe): + """Success case: get path to executable.""" assert ( - model.get_exe_path() - == internal.SRC_DIR / model.name / "offline" / internal.CABLE_EXE + model.get_exe_path(mpi=mpi) + == internal.SRC_DIR / model.name / "offline" / expected_exe ) @@ -142,10 +152,18 @@ def _setup(self, model): (internal.SRC_DIR / model.name / "offline" / "Makefile").touch() (internal.SRC_DIR / model.name / "offline" / "foo.f90").touch() - def test_source_files_and_scripts_are_copied_to_tmp_dir(self, model): + @pytest.fixture() + def tmp_dir(self, model, mpi): + """Return the relative path to the temporary build directory.""" + return ( + internal.SRC_DIR + / model.name + / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR) + ) + + def test_source_files_and_scripts_are_copied_to_tmp_dir(self, model, mpi, tmp_dir): """Success case: test source files and scripts are copied to .tmp.""" - model.pre_build() - tmp_dir = internal.SRC_DIR / model.name / "offline" / ".tmp" + model.pre_build(mpi=mpi) assert (tmp_dir / "Makefile").exists() assert (tmp_dir / "foo.f90").exists() @@ -164,30 +182,36 @@ def modules(self): return ["foo", "bar"] @pytest.fixture() - def env(self, netcdf_root): - """Return a dictionary containing the required environment variables.""" + def expected_env(self, netcdf_root, mpi): + """Return a dictionary of expected environment variables to be defined.""" return { "NCDIR": f"{netcdf_root}/lib/Intel", "NCMOD": f"{netcdf_root}/include/Intel", "CFLAGS": "-O2 -fp-model precise", "LDFLAGS": f"-L{netcdf_root}/lib/Intel -O0", "LD": "-lnetcdf -lnetcdff", - "FC": "ifort", + "FC": "mpif90" if mpi else "ifort", } @pytest.fixture(autouse=True) def _setup(self, model, netcdf_root): """Setup precondition for `Model.run_build()`.""" - (internal.SRC_DIR / model.name / "offline" / ".tmp").mkdir(parents=True) + (internal.SRC_DIR / model.name / internal.TMP_BUILD_DIR).mkdir(parents=True) + (internal.SRC_DIR / model.name / internal.TMP_BUILD_DIR_MPI).mkdir(parents=True) # This is required so that we can use the NETCDF_ROOT environment # variable when running `make`: os.environ["NETCDF_ROOT"] = netcdf_root - def test_build_command_execution(self, model, mock_subprocess_handler, modules): + @pytest.mark.parametrize( + ("mpi", "expected_commands"), [(False, ["make"]), (True, ["make mpi"])] + ) + def test_build_command_execution( + self, model, mock_subprocess_handler, modules, mpi, expected_commands + ): """Success case: test build commands are run.""" - model.run_build(modules) - assert mock_subprocess_handler.commands == ["make"] + model.run_build(modules, mpi=mpi) + assert mock_subprocess_handler.commands == expected_commands def test_modules_loaded_at_runtime( self, model, mock_environment_modules_handler, modules @@ -202,11 +226,11 @@ def test_modules_loaded_at_runtime( ) in mock_environment_modules_handler.commands def test_commands_are_run_with_environment_variables( - self, model, mock_subprocess_handler, modules, env + self, model, mock_subprocess_handler, modules, mpi, expected_env ): """Success case: test commands are run with the correct environment variables.""" - model.run_build(modules) - for kv in env.items(): + model.run_build(modules, mpi=mpi) + for kv in expected_env.items(): assert kv in mock_subprocess_handler.env.items() @@ -216,18 +240,38 @@ class TestPostBuild: @pytest.fixture(autouse=True) def _setup(self, model): """Setup precondition for `Model.post_build()`.""" - (internal.SRC_DIR / model.name / "offline" / ".tmp").mkdir(parents=True) - ( - internal.SRC_DIR / model.name / "offline" / ".tmp" / internal.CABLE_EXE - ).touch() + tmp_build_dir = internal.SRC_DIR / model.name / internal.TMP_BUILD_DIR + tmp_build_dir.mkdir(parents=True) + (tmp_build_dir / internal.CABLE_EXE).touch() + + tmp_build_dir_mpi = internal.SRC_DIR / model.name / internal.TMP_BUILD_DIR_MPI + tmp_build_dir_mpi.mkdir(parents=True) + (tmp_build_dir_mpi / internal.CABLE_MPI_EXE).touch() + + @pytest.fixture() + def tmp_dir(self, model, mpi): + """Return the relative path to the temporary build directory.""" + return ( + internal.SRC_DIR + / model.name + / (internal.TMP_BUILD_DIR_MPI if mpi else internal.TMP_BUILD_DIR) + ) + + @pytest.fixture() + def exe(self, mpi): + """Return the name of the CABLE executable.""" + return internal.CABLE_MPI_EXE if mpi else internal.CABLE_EXE + + @pytest.fixture() + def offline_dir(self, model): + """Return the relative path to the offline source directory.""" + return internal.SRC_DIR / model.name / "offline" - def test_exe_moved_to_offline_dir(self, model): + def test_exe_moved_to_offline_dir(self, model, mpi, tmp_dir, exe, offline_dir): """Success case: test executable is moved to offline directory.""" - model.post_build() - tmp_dir = internal.SRC_DIR / model.name / "offline" / ".tmp" - assert not (tmp_dir / internal.CABLE_EXE).exists() - offline_dir = internal.SRC_DIR / model.name / "offline" - assert (offline_dir / internal.CABLE_EXE).exists() + model.post_build(mpi=mpi) + assert not (tmp_dir / exe).exists() + assert (offline_dir / exe).exists() class TestCustomBuild: diff --git a/tests/test_namelist.py b/tests/test_namelist.py new file mode 100644 index 00000000..87d629cf --- /dev/null +++ b/tests/test_namelist.py @@ -0,0 +1,87 @@ +"""`pytest` tests for namelist.py.""" + +from pathlib import Path + +import f90nml +import pytest + +from benchcab.utils.namelist import patch_namelist, patch_remove_namelist + + +class TestPatchNamelist: + """Tests for `patch_namelist()`.""" + + @pytest.fixture() + def nml_path(self): + """Return a path to a namelist file used for testing.""" + return Path("test.nml") + + def test_patch_on_non_existing_namelist_file(self, nml_path): + """Success case: patch non-existing namelist file.""" + patch = {"cable": {"file": "/path/to/file", "bar": 123}} + patch_namelist(nml_path, patch) + assert f90nml.read(nml_path) == patch + + def test_patch_on_non_empty_namelist_file(self, nml_path): + """Success case: patch non-empty namelist file.""" + f90nml.write({"cable": {"file": "/path/to/file", "bar": 123}}, nml_path) + patch_namelist(nml_path, {"cable": {"some": {"parameter": True}, "bar": 456}}) + assert f90nml.read(nml_path) == { + "cable": { + "file": "/path/to/file", + "bar": 456, + "some": {"parameter": True}, + } + } + + def test_empty_patch_does_nothing(self, nml_path): + """Success case: empty patch does nothing.""" + f90nml.write({"cable": {"file": "/path/to/file", "bar": 123}}, nml_path) + prev = f90nml.read(nml_path) + patch_namelist(nml_path, {}) + assert f90nml.read(nml_path) == prev + + +class TestPatchRemoveNamelist: + """Tests for `patch_remove_namelist()`.""" + + @pytest.fixture() + def nml(self): + """Return a namelist dictionary used for testing.""" + return { + "cable": { + "cable_user": { + "some_parameter": True, + "new_feature": True, + }, + }, + } + + @pytest.fixture() + def nml_path(self, nml): + """Create a namelist file and return its path.""" + _nml_path = Path("test.nml") + f90nml.write(nml, _nml_path) + return _nml_path + + def test_remove_namelist_parameter_from_derived_type(self, nml_path): + """Success case: remove a namelist parameter from derrived type.""" + patch_remove_namelist( + nml_path, {"cable": {"cable_user": {"new_feature": True}}} + ) + assert f90nml.read(nml_path) == { + "cable": {"cable_user": {"some_parameter": True}} + } + + def test_empty_patch_remove_does_nothing(self, nml_path, nml): + """Success case: empty patch_remove does nothing.""" + patch_remove_namelist(nml_path, {}) + assert f90nml.read(nml_path) == nml + + def test_key_error_raised_for_non_existent_namelist_parameter(self, nml_path): + """Failure case: test patch_remove KeyError exeption.""" + with pytest.raises( + KeyError, + match=f"Namelist parameters specified in `patch_remove` do not exist in {nml_path.name}.", + ): + patch_remove_namelist(nml_path, {"cable": {"foo": {"bar": True}}}) diff --git a/tests/test_spatial.py b/tests/test_spatial.py new file mode 100644 index 00000000..b0a006ee --- /dev/null +++ b/tests/test_spatial.py @@ -0,0 +1,221 @@ +"""`pytest` tests for spatial.py. + +Note: explicit teardown for generated files and directories are not required as +the working directory used for testing is cleaned up in the `_run_around_tests` +pytest autouse fixture. +""" + +import contextlib +import io +import logging +from pathlib import Path + +import f90nml +import pytest +import yaml + +from benchcab import internal +from benchcab.model import Model +from benchcab.spatial import SpatialTask, get_spatial_tasks +from benchcab.utils import get_logger +from benchcab.utils.repo import Repo + + +@pytest.fixture() +def mock_repo(): + class MockRepo(Repo): + def __init__(self) -> None: + self.branch = "test-branch" + self.revision = "1234" + + def checkout(self): + pass + + def get_branch_name(self) -> str: + return self.branch + + def get_revision(self) -> str: + return self.revision + + return MockRepo() + + +@pytest.fixture() +def model(mock_subprocess_handler, mock_repo): + """Returns a `Model` instance.""" + _model = Model( + model_id=1, + repo=mock_repo, + patch={"cable": {"some_branch_specific_setting": True}}, + ) + _model.subprocess_handler = mock_subprocess_handler + return _model + + +@pytest.fixture() +def task(model, mock_subprocess_handler): + """Returns a mock `SpatialTask` instance.""" + _task = SpatialTask( + model=model, + met_forcing_name="crujra_access", + met_forcing_payu_experiment="https://github.com/CABLE-LSM/cable_example.git", + sci_conf_id=0, + sci_config={"cable": {"some_setting": True}}, + ) + _task.subprocess_handler = mock_subprocess_handler + return _task + + +class TestGetTaskName: + """Tests for `SpatialTask.get_task_name()`.""" + + def test_task_name_convention(self, task): + """Success case: check task name convention.""" + assert task.get_task_name() == "crujra_access_R1_S0" + + +class TestConfigureExperiment: + """Tests for `SpatialTask.configure_experiment()`.""" + + @pytest.fixture(autouse=True) + def _create_task_dir(self): + task_dir = internal.SPATIAL_TASKS_DIR / "crujra_access_R1_S0" + task_dir.mkdir(parents=True) + (task_dir / "config.yaml").touch() + (task_dir / "cable.nml").touch() + + def test_payu_config_parameters(self, task): + """Success case: check config.yaml parameters.""" + task.configure_experiment(payu_config={"some_parameter": "foo"}) + config_path = internal.SPATIAL_TASKS_DIR / task.get_task_name() / "config.yaml" + with config_path.open("r", encoding="utf-8") as file: + config = yaml.safe_load(file) + assert config["exe"] == str( + ( + internal.SRC_DIR / "test-branch" / "offline" / internal.CABLE_MPI_EXE + ).absolute() + ) + assert config["laboratory"] == str(internal.PAYU_LABORATORY_DIR.absolute()) + assert config["some_parameter"] == "foo" + + @pytest.mark.parametrize( + ("verbosity", "expected"), + [ + (logging.INFO, ""), + ( + logging.DEBUG, + " Updating experiment config parameters in " + "runs/spatial/tasks/crujra_access_R1_S0/config.yaml", + ), + ], + ) + def test_standard_output(self, task, verbosity, expected, caplog): + """Success case: test standard output.""" + caplog.set_level(verbosity) + task.configure_experiment() + output = "\n".join(caplog.messages) if caplog.messages else "" + assert output == expected + + +class TestUpdateNamelist: + """Tests for `SpatialTask.update_namelist()`.""" + + @pytest.fixture(autouse=True) + def _create_task_dir(self): + task_dir = internal.SPATIAL_TASKS_DIR / "crujra_access_R1_S0" + task_dir.mkdir(parents=True) + (task_dir / "config.yaml").touch() + (task_dir / "cable.nml").touch() + + def test_namelist_parameters_are_patched(self, task): + """Success case: test namelist parameters are patched.""" + task.update_namelist() + res_nml = f90nml.read( + str(internal.SPATIAL_TASKS_DIR / task.get_task_name() / internal.CABLE_NML) + ) + assert res_nml["cable"] == { + "some_setting": True, + "some_branch_specific_setting": True, + } + + @pytest.mark.parametrize( + ("verbosity", "expected"), + [ + (logging.INFO, ""), + ( + logging.DEBUG, + " Adding science configurations to CABLE namelist file " + "runs/spatial/tasks/crujra_access_R1_S0/cable.nml\n" + " Adding branch specific configurations to CABLE namelist file " + "runs/spatial/tasks/crujra_access_R1_S0/cable.nml", + ), + ], + ) + def test_standard_output(self, task, verbosity, expected, caplog): + """Success case: test standard output.""" + caplog.set_level(verbosity) + task.update_namelist() + output = "\n".join(caplog.messages) if caplog.messages else "" + assert output == expected + + +class TestRun: + """Tests for `SpatialTask.run()`.""" + + @pytest.fixture(autouse=True) + def _setup(self, task): + task_dir = internal.SPATIAL_TASKS_DIR / task.get_task_name() + task_dir.mkdir(parents=True, exist_ok=True) + + def test_payu_run_command(self, task, mock_subprocess_handler): + """Success case: test payu run command.""" + task.run() + assert "payu run" in mock_subprocess_handler.commands + + def test_payu_run_with_optional_arguments(self, task, mock_subprocess_handler): + """Success case: test payu run command with optional arguments.""" + task.payu_args = "--some-flag" + task.run() + assert "payu run --some-flag" in mock_subprocess_handler.commands + + +class TestGetSpatialTasks: + """Tests for `get_spatial_tasks()`.""" + + @pytest.fixture() + def models(self, mock_repo): + """Return a list of `Model` instances used for testing.""" + return [Model(repo=mock_repo, model_id=id) for id in range(2)] + + @pytest.fixture() + def met_forcings(self, config): + """Return a list of spatial met forcing specifications.""" + return config["spatial"]["met_forcings"] + + @pytest.fixture() + def science_configurations(self, config): + """Return a list of science configurations used for testing.""" + return config["science_configurations"] + + def test_task_product_across_branches_forcings_and_configurations( + self, models, met_forcings, science_configurations + ): + """Success case: test task product across branches, forcings and configurations.""" + tasks = get_spatial_tasks( + models=models, + met_forcings=met_forcings, + science_configurations=science_configurations, + ) + met_forcing_names = list(met_forcings.keys()) + assert [ + (task.model, task.met_forcing_name, task.sci_config) for task in tasks + ] == [ + (models[0], met_forcing_names[0], science_configurations[0]), + (models[0], met_forcing_names[0], science_configurations[1]), + (models[0], met_forcing_names[1], science_configurations[0]), + (models[0], met_forcing_names[1], science_configurations[1]), + (models[1], met_forcing_names[0], science_configurations[0]), + (models[1], met_forcing_names[0], science_configurations[1]), + (models[1], met_forcing_names[1], science_configurations[0]), + (models[1], met_forcing_names[1], science_configurations[1]), + ] diff --git a/tests/test_workdir.py b/tests/test_workdir.py index 62dd710c..ae67676e 100644 --- a/tests/test_workdir.py +++ b/tests/test_workdir.py @@ -12,6 +12,7 @@ from benchcab.workdir import ( clean_directory_tree, setup_fluxsite_directory_tree, + setup_spatial_directory_tree, ) @@ -37,6 +38,25 @@ def test_directory_structure_generated(self, fluxsite_directory_list): assert path.exists() +class TestSetupSpatialDirectoryTree: + """Tests for `setup_spatial_directory_tree()`.""" + + @pytest.fixture() + def spatial_directory_list(self): + """Return the list of work directories we want benchcab to create.""" + return [ + Path("runs", "spatial"), + Path("runs", "spatial", "tasks"), + Path("runs", "payu-laboratory"), + ] + + def test_directory_structure_generated(self, spatial_directory_list): + """Success case: generate spatial directory structure.""" + setup_spatial_directory_tree() + for path in spatial_directory_list: + assert path.exists() + + class TestCleanDirectoryTree: """Tests for `clean_directory_tree()`."""