diff --git a/esm_runscripts/__init__.py b/esm_runscripts/__init__.py index 67d467a..b94e3e1 100644 --- a/esm_runscripts/__init__.py +++ b/esm_runscripts/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = 'dirk.barbi@awi.de' -__version__ = "5.1.25" +__version__ = "5.1.27" from .sim_objects import * from .batch_system import * diff --git a/esm_runscripts/compute.py b/esm_runscripts/compute.py index 1628e79..b087cd8 100644 --- a/esm_runscripts/compute.py +++ b/esm_runscripts/compute.py @@ -284,7 +284,7 @@ def initialize_experiment_logfile(config): log_msg = f"# Beginning of Experiment {config['general']['expid']}" write_to_log(config, [log_msg], message_sep="") - + write_to_log( config, [ @@ -302,7 +302,7 @@ def initialize_experiment_logfile(config): f"{config['general']['experiment_dir']}/log" \ f"/{config['general']['expid']}_esm_runscripts_" \ f"{config['general']['run_datestamp']}.log" - + logger.trace_sink.def_path(logfile_path) return config @@ -318,7 +318,7 @@ def _write_finalized_config(config): # here: https://pyyaml.org/wiki/PyYAMLDocumentation def date_representer(dumper, date): return dumper.represent_str(f"{date.output()}") - + def calendar_representer(dumper, calendar): # Calendar has a __str__ method return dumper.represent_str(str(calendar)) @@ -338,29 +338,29 @@ class EsmConfigDumper(yaml.dumper.Dumper): pass # pyyaml does not support tuple and prints !!python/tuple - EsmConfigDumper.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list) + EsmConfigDumper.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list) # Determine how non-built-in types will be printed be the YAML dumper - EsmConfigDumper.add_representer(esm_calendar.Date, date_representer) - - EsmConfigDumper.add_representer(esm_calendar.esm_calendar.Calendar, - calendar_representer) - # yaml.representer.SafeRepresenter.represent_str) - - EsmConfigDumper.add_representer(esm_parser.esm_parser.ConfigSetup, - yaml.representer.SafeRepresenter.represent_dict) - + EsmConfigDumper.add_representer(esm_calendar.Date, date_representer) + + EsmConfigDumper.add_representer(esm_calendar.esm_calendar.Calendar, + calendar_representer) + # yaml.representer.SafeRepresenter.represent_str) + + EsmConfigDumper.add_representer(esm_parser.esm_parser.ConfigSetup, + yaml.representer.SafeRepresenter.represent_dict) + EsmConfigDumper.add_representer(batch_system, batch_system_representer) # format for the other ESM data structures - EsmConfigDumper.add_representer(esm_rcfile.esm_rcfile.EsmToolsDir, - yaml.representer.SafeRepresenter.represent_str) - - EsmConfigDumper.add_representer(esm_runscripts.coupler.coupler_class, - coupler_representer) - - EsmConfigDumper.add_representer(esm_runscripts.oasis.oasis, oasis_representer) - + EsmConfigDumper.add_representer(esm_rcfile.esm_rcfile.EsmToolsDir, + yaml.representer.SafeRepresenter.represent_str) + + EsmConfigDumper.add_representer(esm_runscripts.coupler.coupler_class, + coupler_representer) + + EsmConfigDumper.add_representer(esm_runscripts.oasis.oasis, oasis_representer) + config_file_path = \ f"{config['general']['thisrun_config_dir']}"\ f"/{config['general']['expid']}_finished_config.yaml" @@ -369,11 +369,11 @@ class EsmConfigDumper(yaml.dumper.Dumper): config_final = copy.deepcopy(config) #PrevRunInfo del config_final["prev_run"] #PrevRunInfo - out = yaml.dump(config_final, Dumper=EsmConfigDumper, width=10000, + out = yaml.dump(config_final, Dumper=EsmConfigDumper, width=10000, indent=4) #PrevRunInfo config_file.write(out) return config - + def color_diff(diff): for line in diff: @@ -417,7 +417,7 @@ def update_runscript(fromdir, scriptsdir, tfile, gconfig, file_type): # if `tfile` contains a full path of the runscript then remove the leading path tfile = os.path.basename(tfile) - + # If the target file in ``scriptsdir`` does not exist, then copy the file # to the target. if not os.path.isfile(scriptsdir + "/" + tfile): @@ -553,7 +553,7 @@ def copy_tools_to_thisrun(config): # protect such problems scriptsdir_deep_parents = list(pathlib.Path(scriptsdir).parents)[5:] deep_nesting_found = pathlib.Path(expdir) in scriptsdir_deep_parents - if deep_nesting_found: + if deep_nesting_found: error_type = "runtime error" error_text = ( f"deep recursion is detected in {__file__}:\n" @@ -562,10 +562,10 @@ def copy_tools_to_thisrun(config): f"- experiment dir: {expdir}" ) # exit right away to prevent further recursion. There might still be - # running instances of esmr_runscripts and something like + # running instances of esmr_runscripts and something like # `killall esm_runscripts` might be required esm_parser.user_error(error_type, error_text) - + # If ``fromdir`` and ``scriptsdir`` are the same, this is already a computing # simulation which means we want to use the script in the experiment folder, # so no copying is needed @@ -598,7 +598,7 @@ def copy_tools_to_thisrun(config): options_to_remove = [" -U ", " --update "] for option in options_to_remove: original_command = original_command.replace(option, " ") - + # Before resubmitting the esm_runscripts, the path of the runscript # needs to be modified. Remove the absolute/relative path runscript_absdir, runscript = os.path.split(gconfig['runscript_abspath']) @@ -606,20 +606,20 @@ def copy_tools_to_thisrun(config): new_command_list = [] for command in original_command_list: # current command will contain the full path, so replace it with - # the YAML file only since we are going to execute it from the + # the YAML file only since we are going to execute it from the # `scriptsdir` now if runscript in command: # gconfig['scriptname'] or `runscript` only contains the YAML file name - command = runscript + command = runscript new_command_list.append(command) new_command = " ".join(new_command_list) restart_command = f"cd {scriptsdir}; esm_runscripts {new_command}" - + # prevent continuous addition of --no-motd if not "--no-motd" in restart_command: restart_command += " --no-motd " - + if config["general"]["verbose"]: print(restart_command) os.system(restart_command) @@ -635,7 +635,7 @@ def _copy_preliminary_files_from_experiment_to_thisrun(config): f"{config['general']['expid']}_{config['general']['setup_name']}.date", "copy", )] - + for filetype, filename, copy_or_link in filelist: source = config["general"]["experiment_" + filetype + "_dir"] dest = config["general"]["thisrun_" + filetype + "_dir"] @@ -670,4 +670,4 @@ def _show_simulation_info(config): six.print_(80 * "=") six.print_() return config - + diff --git a/esm_runscripts/filelists.py b/esm_runscripts/filelists.py index 2dec4f0..e880cac 100644 --- a/esm_runscripts/filelists.py +++ b/esm_runscripts/filelists.py @@ -632,11 +632,13 @@ def copy_files(config, filetypes, source, target): for filetype in [filetype for filetype in filetypes if not filetype == "ignore"]: for model in config["general"]["valid_model_names"] + ["general"]: - movement_method = get_method(get_movement(config, model, filetype, source, target)) if filetype + "_" + text_source in config[model]: sourceblock = config[model][filetype + "_" + text_source] targetblock = config[model][filetype + "_" + text_target] for categ in sourceblock: + movement_method = get_method( + get_movement(config, model, categ, filetype, source, target) + ) file_source = os.path.normpath(sourceblock[categ]) file_target = os.path.normpath(targetblock[categ]) if config["general"]["verbose"]: @@ -801,6 +803,40 @@ def complete_all_file_movements(config): config = complete_one_file_movement(config, model, filetype, movement, movement_type) del mconfig["file_movements"][filetype]["all_directions"] + # Complete file specific movements with ``all_directions`` + for file_in_fm in mconfig["file_movements"]: + # If it is a specific file, and not a file type + if file_in_fm not in ( + config["general"]["all_model_filetypes"] + + ["scripts", "unknown"] + ): + # Check syntax for restart files + if ( + file_in_fm in mconfig.get("restart_in_files", {}) + or file_in_fm in mconfig.get("restart_out_files", {}) + ): + esm_parser.user_error( + "Movement direction not specified", + f"'{model}.file_movements.{file_in_fm}' refers to a " + + "restart file which can be moved/copied/link in two " + + "directions, into the 'work' folder and out of the " + + "'work' folder. Please, add the direction '_in' or " + + f"'_out' to '{file_in_fm}':\n\n{model}:\n " + + f"file_movements:\n {file_in_fm}_:\n" + + f" [ ... ]" + ) + # Solve ``all_directions`` + file_spec_movements = mconfig["file_movements"][file_in_fm] + if "all_directions" in file_spec_movements: + movement_type = file_spec_movements["all_directions"] + for movement in [ + 'init_to_exp', 'exp_to_run', 'run_to_work', 'work_to_run' + ]: + config = complete_one_file_movement( + config, model, file_in_fm, movement, movement_type + ) + del mconfig["file_movements"][file_in_fm]["all_directions"] + if "default" in mconfig["file_movements"]: if "all_directions" in mconfig["file_movements"]["default"]: movement_type = mconfig["file_movements"]["default"]["all_directions"] @@ -816,7 +852,20 @@ def complete_all_file_movements(config): return config -def get_movement(config, model, filetype, source, target): +def get_movement(config, model, categ, filetype, source, target): + # Remove globing strings from categ + if isinstance(categ, str): + categ = categ.split("_glob_")[0] + # Two type of directions are needed for restarts, therefore, the categories need an + # "_in" or "_out" at the end. + if filetype=="restart_in": + categ = f"{categ}_in" + elif filetype=="restart_out": + categ = f"{categ}_out" + # File specific movements + file_spec_movements = config[model]["file_movements"].get(categ, {}) + # Movements associated to ``filetypes`` + file_type_movements = config[model]["file_movements"][filetype] if source == "init": # Get the model-specific reusable_filetypes, if not existing, get the # general ones @@ -825,13 +874,25 @@ def get_movement(config, model, filetype, source, target): config["general"]["reusable_filetypes"] ) if config["general"]["run_number"] == 1 or filetype not in model_reusable_filetypes: - return config[model]["file_movements"][filetype]["init_to_exp"] + return file_spec_movements.get( + "init_to_exp", + file_type_movements["init_to_exp"] + ) else: - return config[model]["file_movements"][filetype]["exp_to_run"] + return file_spec_movements.get( + "exp_to_run", + file_type_movements["exp_to_run"] + ) elif source == "work": - return config[model]["file_movements"][filetype]["work_to_run"] + return file_spec_movements.get( + "work_to_run", + file_type_movements["work_to_run"] + ) elif source == "thisrun" and target == "work": - return config[model]["file_movements"][filetype]["run_to_work"] + return file_spec_movements.get( + "run_to_work", + file_type_movements["run_to_work"] + ) else: # This should NOT happen print(f"Error: Unknown file movement from {source} to {target}", flush=True) diff --git a/esm_runscripts/namelists.py b/esm_runscripts/namelists.py index 4313ad8..3e0fe8a 100644 --- a/esm_runscripts/namelists.py +++ b/esm_runscripts/namelists.py @@ -149,7 +149,7 @@ def nmls_remove(mconfig): del namelist_changes[namelist][change_chapter][key] if remove_original_key: del namelist_changes[namelist][change_chapter][original_key] - + # mconfig instead of config, Grrrrr print(f"- NOTE: removing the variable: {key} from the namelist: {namelist}") @@ -322,7 +322,7 @@ def nmls_output(mconfig): print(f'::: end of the contents of {nml_name}\n') return mconfig - + @staticmethod def nmls_output_all(config): six.print_( @@ -342,6 +342,6 @@ def __init__(self, *args, **kwargs): DeprecationWarning, stacklevel=2, ) - + super(namelist, self).__init__(*args, **kwargs) diff --git a/esm_runscripts/prepare.py b/esm_runscripts/prepare.py index 388d7b5..705f0d4 100644 --- a/esm_runscripts/prepare.py +++ b/esm_runscripts/prepare.py @@ -28,6 +28,88 @@ def mini_resolve_variable_date_file(date_file, config): return date_file +def maybe_add_extra_echam_streams(config): + import f90nml + import os + + if "echam" in config["general"]["valid_model_names"]: + mconfig = config["echam"] + if not mconfig.get("determine_echam_streams_from_namelist", False): + return config + + template_namelist_dir = mconfig["namelist_dir"] + + # Only solve my edge case for right now. Normally namelsist dir should + # be a full thing. + # + # This is horrible. Just horrible. + if "${project_base}" in template_namelist_dir: + template_namelist_dir = template_namelist_dir.replace("${project_base}", config["general"]["project_base"]) + + nml = f90nml.read(os.path.join(template_namelist_dir, "namelist.echam")) + mvstreams = nml.get("mvstreamctl") + # If only one mvstreamctl is there, it comes back as a namelist, but we + # want lists: + if not isinstance(mvstreams, list): + mvstreams = [mvstreams] + extra_streams = [] + for mvstream in mvstreams: + if mvstream is not None: + filetag = mvstream.get("filetag") + if filetag: + extra_streams.append(filetag) + else: + target = mvstream.get("target", "*m") # According to the handbook, the default is "*m" if not set + if target.startswith("*"): + ending = target[1:] + source = mvstream.get("source") # NOTE(PG): Are these ever lists?? + if isinstance(source, str): + extra_streams.append(source+ending) + elif isinstance(source, list): + sources = source # Pluralize the variable name to make it more obvious we have a list + for source in sources: + extra_streams.append(source+ending) + else: + extra_streams.append(target) + set_streams = nml.get("set_stream") + # If only one set_stream is there, it comes back as a namelist, but we + # want lists: + if not isinstance(set_streams, list): + set_streams = [set_streams] + for set_stream in set_streams: + if set_stream is not None: + if set_stream.get("lpost", 1): # 1 is the safer option. In the worst case, esm-tools complain about not being to move some files + if set_stream.get("post_suf"): + # User has defined their own suffix, we need to add that to the list: + extra_streams.append(set_stream.get("post_suf")) + else: + # User did not define a own suffix, but the default (if I understand the handbook) is to just use the stream name: + extra_streams.append(set_stream.get("stream")) + # Apparently you can also rename the restart files uniquely + if set_stream.get("rest_suf"): + extra_streams.append(set_stream.get("rest_suf")) + set_stream_elements = nml.get("set_stream_element") + # If only one set_stream_elements is there, it comes back as a namelist, but we + # want lists: + if not isinstance(set_stream_elements, list): + set_stream_elements = [set_stream_elements] + for set_stream_element in set_stream_elements: + if set_stream_element is not None: + lpost = set_stream_element.get("lpost", 1) + lrerun = set_stream_element.get("lrerun", 1) + stream = set_stream_element['stream'] # Will ECHAM crash if there is no stream in one of these chapter??? + if lpost or lrerun: + extra_streams.append(stream) + + # Remove duplicates: + extra_streams = list(set(extra_streams)) + for stream in extra_streams: + if stream not in config['echam']['streams']: + print(f"NOTE:\t\tAdding the following stream detected in namelist.echam which was not declared in echam.yaml\n \t\t{stream}") + config['echam']['streams'].append(stream) + return config + + def _read_date_file(config): import os import logging diff --git a/setup.cfg b/setup.cfg index 75f2cc6..3a70400 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.1.25 +current_version = 5.1.27 commit = True tag = True diff --git a/setup.py b/setup.py index fdcf430..4cdb9d8 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,6 @@ test_suite='tests', tests_require=test_requirements, url='https://github.com/esm-tools/esm_runscripts', - version="5.1.25", + version="5.1.27", zip_safe=False, )