From 0dd0bc45c1f5877dd963815bbf71e787338ab82e Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Wed, 28 Jul 2021 10:00:33 +0200 Subject: [PATCH 01/11] feat(namelist): allows user to override streams by checking what is defined in the namelist --- esm_runscripts/compute.py | 3 +++ esm_runscripts/namelists.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/esm_runscripts/compute.py b/esm_runscripts/compute.py index 1628e79..500cf95 100644 --- a/esm_runscripts/compute.py +++ b/esm_runscripts/compute.py @@ -176,6 +176,9 @@ def modify_namelists(config): config[model] = Namelist.nmls_remove(config[model]) if model == "echam": config = Namelist.apply_echam_disturbance(config) + # NOTE(PG): This really doesn't belong in modify namelists, but, + # what the hell... + config = Namelist.echam_determine_streams_from_nml(config) config[model] = Namelist.nmls_modify(config[model]) config[model] = Namelist.nmls_finalize( config[model], config["general"]["verbose"] diff --git a/esm_runscripts/namelists.py b/esm_runscripts/namelists.py index 4313ad8..f1604ec 100644 --- a/esm_runscripts/namelists.py +++ b/esm_runscripts/namelists.py @@ -261,6 +261,24 @@ def apply_echam_disturbance(config): ) return config + @staticmethod + def echam_determine_streams_from_nml(config): + if "echam" in config["general"]["valid_model_names"]: + nml = config["echam"]["namelists"]["namelist.echam"] + mvstreams = nml["mvstreamctl"] + mvstreams_tags = [nml["filetag"] for nml in mvstreams] + # NOTE(PG): There may still be warnings about missing files -- we + # still need to implement an "allowed missing files" feature, but + # this should now put all the streams away correctly. + if not config["echam"].get("override_streams_from_namelist", False): + config["echam"]["streams"] += mvstreams_tags + else: + # NOTE(PG): I honestly am not sure if this will work, maybe the + # restart will get messed up horribly. This just overrides + # whatever was there in the default. It might be dangerous. + config["echam"]["streams"] = mvstreams_tags + return config + @staticmethod def nmls_finalize(mconfig, verbose): """ From 6df978cf41b696d546ff2c6726b05e1ef5e0a038 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Wed, 28 Jul 2021 15:46:24 +0200 Subject: [PATCH 02/11] style(whitespace): guys....fix your editors --- esm_runscripts/compute.py | 68 ++++++++++++++++++------------------- esm_runscripts/namelists.py | 6 ++-- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/esm_runscripts/compute.py b/esm_runscripts/compute.py index 500cf95..d20a16c 100644 --- a/esm_runscripts/compute.py +++ b/esm_runscripts/compute.py @@ -287,7 +287,7 @@ def initialize_experiment_logfile(config): log_msg = f"# Beginning of Experiment {config['general']['expid']}" write_to_log(config, [log_msg], message_sep="") - + write_to_log( config, [ @@ -305,7 +305,7 @@ def initialize_experiment_logfile(config): f"{config['general']['experiment_dir']}/log" \ f"/{config['general']['expid']}_esm_runscripts_" \ f"{config['general']['run_datestamp']}.log" - + logger.trace_sink.def_path(logfile_path) return config @@ -321,7 +321,7 @@ def _write_finalized_config(config): # here: https://pyyaml.org/wiki/PyYAMLDocumentation def date_representer(dumper, date): return dumper.represent_str(f"{date.output()}") - + def calendar_representer(dumper, calendar): # Calendar has a __str__ method return dumper.represent_str(str(calendar)) @@ -341,29 +341,29 @@ class EsmConfigDumper(yaml.dumper.Dumper): pass # pyyaml does not support tuple and prints !!python/tuple - EsmConfigDumper.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list) + EsmConfigDumper.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list) # Determine how non-built-in types will be printed be the YAML dumper - EsmConfigDumper.add_representer(esm_calendar.Date, date_representer) - - EsmConfigDumper.add_representer(esm_calendar.esm_calendar.Calendar, - calendar_representer) - # yaml.representer.SafeRepresenter.represent_str) - - EsmConfigDumper.add_representer(esm_parser.esm_parser.ConfigSetup, - yaml.representer.SafeRepresenter.represent_dict) - + EsmConfigDumper.add_representer(esm_calendar.Date, date_representer) + + EsmConfigDumper.add_representer(esm_calendar.esm_calendar.Calendar, + calendar_representer) + # yaml.representer.SafeRepresenter.represent_str) + + EsmConfigDumper.add_representer(esm_parser.esm_parser.ConfigSetup, + yaml.representer.SafeRepresenter.represent_dict) + EsmConfigDumper.add_representer(batch_system, batch_system_representer) # format for the other ESM data structures - EsmConfigDumper.add_representer(esm_rcfile.esm_rcfile.EsmToolsDir, - yaml.representer.SafeRepresenter.represent_str) - - EsmConfigDumper.add_representer(esm_runscripts.coupler.coupler_class, - coupler_representer) - - EsmConfigDumper.add_representer(esm_runscripts.oasis.oasis, oasis_representer) - + EsmConfigDumper.add_representer(esm_rcfile.esm_rcfile.EsmToolsDir, + yaml.representer.SafeRepresenter.represent_str) + + EsmConfigDumper.add_representer(esm_runscripts.coupler.coupler_class, + coupler_representer) + + EsmConfigDumper.add_representer(esm_runscripts.oasis.oasis, oasis_representer) + config_file_path = \ f"{config['general']['thisrun_config_dir']}"\ f"/{config['general']['expid']}_finished_config.yaml" @@ -372,11 +372,11 @@ class EsmConfigDumper(yaml.dumper.Dumper): config_final = copy.deepcopy(config) #PrevRunInfo del config_final["prev_run"] #PrevRunInfo - out = yaml.dump(config_final, Dumper=EsmConfigDumper, width=10000, + out = yaml.dump(config_final, Dumper=EsmConfigDumper, width=10000, indent=4) #PrevRunInfo config_file.write(out) return config - + def color_diff(diff): for line in diff: @@ -420,7 +420,7 @@ def update_runscript(fromdir, scriptsdir, tfile, gconfig, file_type): # if `tfile` contains a full path of the runscript then remove the leading path tfile = os.path.basename(tfile) - + # If the target file in ``scriptsdir`` does not exist, then copy the file # to the target. if not os.path.isfile(scriptsdir + "/" + tfile): @@ -556,7 +556,7 @@ def copy_tools_to_thisrun(config): # protect such problems scriptsdir_deep_parents = list(pathlib.Path(scriptsdir).parents)[5:] deep_nesting_found = pathlib.Path(expdir) in scriptsdir_deep_parents - if deep_nesting_found: + if deep_nesting_found: error_type = "runtime error" error_text = ( f"deep recursion is detected in {__file__}:\n" @@ -565,10 +565,10 @@ def copy_tools_to_thisrun(config): f"- experiment dir: {expdir}" ) # exit right away to prevent further recursion. There might still be - # running instances of esmr_runscripts and something like + # running instances of esmr_runscripts and something like # `killall esm_runscripts` might be required esm_parser.user_error(error_type, error_text) - + # If ``fromdir`` and ``scriptsdir`` are the same, this is already a computing # simulation which means we want to use the script in the experiment folder, # so no copying is needed @@ -601,7 +601,7 @@ def copy_tools_to_thisrun(config): options_to_remove = [" -U ", " --update "] for option in options_to_remove: original_command = original_command.replace(option, " ") - + # Before resubmitting the esm_runscripts, the path of the runscript # needs to be modified. Remove the absolute/relative path runscript_absdir, runscript = os.path.split(gconfig['runscript_abspath']) @@ -609,20 +609,20 @@ def copy_tools_to_thisrun(config): new_command_list = [] for command in original_command_list: # current command will contain the full path, so replace it with - # the YAML file only since we are going to execute it from the + # the YAML file only since we are going to execute it from the # `scriptsdir` now if runscript in command: # gconfig['scriptname'] or `runscript` only contains the YAML file name - command = runscript + command = runscript new_command_list.append(command) new_command = " ".join(new_command_list) restart_command = f"cd {scriptsdir}; esm_runscripts {new_command}" - + # prevent continuous addition of --no-motd if not "--no-motd" in restart_command: restart_command += " --no-motd " - + if config["general"]["verbose"]: print(restart_command) os.system(restart_command) @@ -638,7 +638,7 @@ def _copy_preliminary_files_from_experiment_to_thisrun(config): f"{config['general']['expid']}_{config['general']['setup_name']}.date", "copy", )] - + for filetype, filename, copy_or_link in filelist: source = config["general"]["experiment_" + filetype + "_dir"] dest = config["general"]["thisrun_" + filetype + "_dir"] @@ -673,4 +673,4 @@ def _show_simulation_info(config): six.print_(80 * "=") six.print_() return config - + diff --git a/esm_runscripts/namelists.py b/esm_runscripts/namelists.py index f1604ec..b6fb0f9 100644 --- a/esm_runscripts/namelists.py +++ b/esm_runscripts/namelists.py @@ -149,7 +149,7 @@ def nmls_remove(mconfig): del namelist_changes[namelist][change_chapter][key] if remove_original_key: del namelist_changes[namelist][change_chapter][original_key] - + # mconfig instead of config, Grrrrr print(f"- NOTE: removing the variable: {key} from the namelist: {namelist}") @@ -340,7 +340,7 @@ def nmls_output(mconfig): print(f'::: end of the contents of {nml_name}\n') return mconfig - + @staticmethod def nmls_output_all(config): six.print_( @@ -360,6 +360,6 @@ def __init__(self, *args, **kwargs): DeprecationWarning, stacklevel=2, ) - + super(namelist, self).__init__(*args, **kwargs) From 26aaa6563ba257898ed8453d6234275d68698bd5 Mon Sep 17 00:00:00 2001 From: Miguel Andres-Martinez Date: Wed, 28 Jul 2021 17:34:14 +0200 Subject: [PATCH 03/11] possibility to define file movements specific to each file --- esm_runscripts/filelists.py | 48 ++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/esm_runscripts/filelists.py b/esm_runscripts/filelists.py index 2dec4f0..d1be16f 100644 --- a/esm_runscripts/filelists.py +++ b/esm_runscripts/filelists.py @@ -632,11 +632,13 @@ def copy_files(config, filetypes, source, target): for filetype in [filetype for filetype in filetypes if not filetype == "ignore"]: for model in config["general"]["valid_model_names"] + ["general"]: - movement_method = get_method(get_movement(config, model, filetype, source, target)) if filetype + "_" + text_source in config[model]: sourceblock = config[model][filetype + "_" + text_source] targetblock = config[model][filetype + "_" + text_target] for categ in sourceblock: + movement_method = get_method( + get_movement(config, model, categ, filetype, source, target) + ) file_source = os.path.normpath(sourceblock[categ]) file_target = os.path.normpath(targetblock[categ]) if config["general"]["verbose"]: @@ -816,7 +818,29 @@ def complete_all_file_movements(config): return config -def get_movement(config, model, filetype, source, target): +def get_movement(config, model, categ, filetype, source, target): + # Remove globing strings from categ + if isinstance(categ, str): + categ = categ.split("_glob_")[0] + # Two type of directions are needed for restarts, therefore, the categories need an + # "_in" or "_out" at the end. They should be defined like that also in the + # ``file_movements`` if specific files are treated differently, in a similar way + # in which ``restart_in`` and ``restart_out`` need their own ``file_movements``. + if filetype=="restart_in": + categ = f"{categ}_in" + elif filetype=="restart_out": + categ = f"{categ}_out" + # Check for endings TODO + # Get file specific movements and complete with ``all_directions`` + file_spec_movements = config[model]["file_movements"].get(categ, {}) + if "all_directions" in file_spec_movements: + movement_type = file_spec_movements["all_directions"] + for movement in ['init_to_exp', 'exp_to_run', 'run_to_work', 'work_to_run']: + config = complete_one_file_movement(config, model, categ, movement, movement_type) + del config[model]["file_movements"][categ]["all_directions"] + + # Movements associated to ``filetypes`` + file_type_movements = config[model]["file_movements"][filetype] if source == "init": # Get the model-specific reusable_filetypes, if not existing, get the # general ones @@ -825,13 +849,25 @@ def get_movement(config, model, filetype, source, target): config["general"]["reusable_filetypes"] ) if config["general"]["run_number"] == 1 or filetype not in model_reusable_filetypes: - return config[model]["file_movements"][filetype]["init_to_exp"] + return file_spec_movements.get( + "init_to_exp", + file_type_movements["init_to_exp"] + ) else: - return config[model]["file_movements"][filetype]["exp_to_run"] + return file_spec_movements.get( + "exp_to_run", + file_type_movements["exp_to_run"] + ) elif source == "work": - return config[model]["file_movements"][filetype]["work_to_run"] + return file_spec_movements.get( + "work_to_run", + file_type_movements["work_to_run"] + ) elif source == "thisrun" and target == "work": - return config[model]["file_movements"][filetype]["run_to_work"] + return file_spec_movements.get( + "run_to_work", + file_type_movements["run_to_work"] + ) else: # This should NOT happen print(f"Error: Unknown file movement from {source} to {target}", flush=True) From 4a434682c72d5632974de7e5f03a4f2b72643ac9 Mon Sep 17 00:00:00 2001 From: Miguel Andres-Martinez Date: Thu, 29 Jul 2021 09:54:52 +0200 Subject: [PATCH 04/11] file specific treatment of the all_directions moved to the complete_all_file_movements method and syntax check added --- esm_runscripts/filelists.py | 47 ++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/esm_runscripts/filelists.py b/esm_runscripts/filelists.py index d1be16f..e880cac 100644 --- a/esm_runscripts/filelists.py +++ b/esm_runscripts/filelists.py @@ -803,6 +803,40 @@ def complete_all_file_movements(config): config = complete_one_file_movement(config, model, filetype, movement, movement_type) del mconfig["file_movements"][filetype]["all_directions"] + # Complete file specific movements with ``all_directions`` + for file_in_fm in mconfig["file_movements"]: + # If it is a specific file, and not a file type + if file_in_fm not in ( + config["general"]["all_model_filetypes"] + + ["scripts", "unknown"] + ): + # Check syntax for restart files + if ( + file_in_fm in mconfig.get("restart_in_files", {}) + or file_in_fm in mconfig.get("restart_out_files", {}) + ): + esm_parser.user_error( + "Movement direction not specified", + f"'{model}.file_movements.{file_in_fm}' refers to a " + + "restart file which can be moved/copied/link in two " + + "directions, into the 'work' folder and out of the " + + "'work' folder. Please, add the direction '_in' or " + + f"'_out' to '{file_in_fm}':\n\n{model}:\n " + + f"file_movements:\n {file_in_fm}_:\n" + + f" [ ... ]" + ) + # Solve ``all_directions`` + file_spec_movements = mconfig["file_movements"][file_in_fm] + if "all_directions" in file_spec_movements: + movement_type = file_spec_movements["all_directions"] + for movement in [ + 'init_to_exp', 'exp_to_run', 'run_to_work', 'work_to_run' + ]: + config = complete_one_file_movement( + config, model, file_in_fm, movement, movement_type + ) + del mconfig["file_movements"][file_in_fm]["all_directions"] + if "default" in mconfig["file_movements"]: if "all_directions" in mconfig["file_movements"]["default"]: movement_type = mconfig["file_movements"]["default"]["all_directions"] @@ -823,22 +857,13 @@ def get_movement(config, model, categ, filetype, source, target): if isinstance(categ, str): categ = categ.split("_glob_")[0] # Two type of directions are needed for restarts, therefore, the categories need an - # "_in" or "_out" at the end. They should be defined like that also in the - # ``file_movements`` if specific files are treated differently, in a similar way - # in which ``restart_in`` and ``restart_out`` need their own ``file_movements``. + # "_in" or "_out" at the end. if filetype=="restart_in": categ = f"{categ}_in" elif filetype=="restart_out": categ = f"{categ}_out" - # Check for endings TODO - # Get file specific movements and complete with ``all_directions`` + # File specific movements file_spec_movements = config[model]["file_movements"].get(categ, {}) - if "all_directions" in file_spec_movements: - movement_type = file_spec_movements["all_directions"] - for movement in ['init_to_exp', 'exp_to_run', 'run_to_work', 'work_to_run']: - config = complete_one_file_movement(config, model, categ, movement, movement_type) - del config[model]["file_movements"][categ]["all_directions"] - # Movements associated to ``filetypes`` file_type_movements = config[model]["file_movements"][filetype] if source == "init": From c0af81a0316e3f3bf82aeb89e949b8785fbc6ce2 Mon Sep 17 00:00:00 2001 From: Miguel Andres-Martinez Date: Thu, 29 Jul 2021 14:14:51 +0200 Subject: [PATCH 05/11] fixed oasis yaml dumper problem for runs that don't use oasis --- esm_runscripts/compute.py | 45 ++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/esm_runscripts/compute.py b/esm_runscripts/compute.py index 1628e79..481105f 100644 --- a/esm_runscripts/compute.py +++ b/esm_runscripts/compute.py @@ -302,7 +302,7 @@ def initialize_experiment_logfile(config): f"{config['general']['experiment_dir']}/log" \ f"/{config['general']['expid']}_esm_runscripts_" \ f"{config['general']['run_datestamp']}.log" - + logger.trace_sink.def_path(logfile_path) return config @@ -318,7 +318,7 @@ def _write_finalized_config(config): # here: https://pyyaml.org/wiki/PyYAMLDocumentation def date_representer(dumper, date): return dumper.represent_str(f"{date.output()}") - + def calendar_representer(dumper, calendar): # Calendar has a __str__ method return dumper.represent_str(str(calendar)) @@ -338,29 +338,30 @@ class EsmConfigDumper(yaml.dumper.Dumper): pass # pyyaml does not support tuple and prints !!python/tuple - EsmConfigDumper.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list) + EsmConfigDumper.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list) # Determine how non-built-in types will be printed be the YAML dumper - EsmConfigDumper.add_representer(esm_calendar.Date, date_representer) - - EsmConfigDumper.add_representer(esm_calendar.esm_calendar.Calendar, - calendar_representer) - # yaml.representer.SafeRepresenter.represent_str) - - EsmConfigDumper.add_representer(esm_parser.esm_parser.ConfigSetup, - yaml.representer.SafeRepresenter.represent_dict) - + EsmConfigDumper.add_representer(esm_calendar.Date, date_representer) + + EsmConfigDumper.add_representer(esm_calendar.esm_calendar.Calendar, + calendar_representer) + # yaml.representer.SafeRepresenter.represent_str) + + EsmConfigDumper.add_representer(esm_parser.esm_parser.ConfigSetup, + yaml.representer.SafeRepresenter.represent_dict) + EsmConfigDumper.add_representer(batch_system, batch_system_representer) # format for the other ESM data structures - EsmConfigDumper.add_representer(esm_rcfile.esm_rcfile.EsmToolsDir, - yaml.representer.SafeRepresenter.represent_str) - - EsmConfigDumper.add_representer(esm_runscripts.coupler.coupler_class, - coupler_representer) - - EsmConfigDumper.add_representer(esm_runscripts.oasis.oasis, oasis_representer) - + EsmConfigDumper.add_representer(esm_rcfile.esm_rcfile.EsmToolsDir, + yaml.representer.SafeRepresenter.represent_str) + + EsmConfigDumper.add_representer(esm_runscripts.coupler.coupler_class, + coupler_representer) + + if "oasis3mct" in config: + EsmConfigDumper.add_representer(esm_runscripts.oasis.oasis, oasis_representer) + config_file_path = \ f"{config['general']['thisrun_config_dir']}"\ f"/{config['general']['expid']}_finished_config.yaml" @@ -369,11 +370,11 @@ class EsmConfigDumper(yaml.dumper.Dumper): config_final = copy.deepcopy(config) #PrevRunInfo del config_final["prev_run"] #PrevRunInfo - out = yaml.dump(config_final, Dumper=EsmConfigDumper, width=10000, + out = yaml.dump(config_final, Dumper=EsmConfigDumper, width=10000, indent=4) #PrevRunInfo config_file.write(out) return config - + def color_diff(diff): for line in diff: From e55adfce7f10c99d0d338b04edb21ad6773226f0 Mon Sep 17 00:00:00 2001 From: Miguel Andres-Martinez Date: Thu, 29 Jul 2021 15:31:31 +0200 Subject: [PATCH 06/11] =?UTF-8?q?Bump=20version:=205.1.25=20=E2=86=92=205.?= =?UTF-8?q?1.26?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- esm_runscripts/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esm_runscripts/__init__.py b/esm_runscripts/__init__.py index 67d467a..d25648e 100644 --- a/esm_runscripts/__init__.py +++ b/esm_runscripts/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = 'dirk.barbi@awi.de' -__version__ = "5.1.25" +__version__ = "5.1.26" from .sim_objects import * from .batch_system import * diff --git a/setup.cfg b/setup.cfg index 75f2cc6..8f27dc8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.1.25 +current_version = 5.1.26 commit = True tag = True diff --git a/setup.py b/setup.py index fdcf430..d27d5a2 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,6 @@ test_suite='tests', tests_require=test_requirements, url='https://github.com/esm-tools/esm_runscripts', - version="5.1.25", + version="5.1.26", zip_safe=False, ) From d9ceb25546f4850de31979f9015799a87c45e334 Mon Sep 17 00:00:00 2001 From: Miguel Andres-Martinez Date: Fri, 30 Jul 2021 14:56:15 +0200 Subject: [PATCH 07/11] =?UTF-8?q?Bump=20version:=205.1.26=20=E2=86=92=205.?= =?UTF-8?q?1.27?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- esm_runscripts/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esm_runscripts/__init__.py b/esm_runscripts/__init__.py index d25648e..b94e3e1 100644 --- a/esm_runscripts/__init__.py +++ b/esm_runscripts/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = 'dirk.barbi@awi.de' -__version__ = "5.1.26" +__version__ = "5.1.27" from .sim_objects import * from .batch_system import * diff --git a/setup.cfg b/setup.cfg index 8f27dc8..3a70400 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.1.26 +current_version = 5.1.27 commit = True tag = True diff --git a/setup.py b/setup.py index d27d5a2..4cdb9d8 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,6 @@ test_suite='tests', tests_require=test_requirements, url='https://github.com/esm-tools/esm_runscripts', - version="5.1.26", + version="5.1.27", zip_safe=False, ) From ea3a0deaf919ddaa4fc192c2f858afc8dc83c819 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Wed, 4 Aug 2021 17:28:20 +0200 Subject: [PATCH 08/11] fix(namelist): the most echam specific thing ever --- esm_runscripts/namelists.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/esm_runscripts/namelists.py b/esm_runscripts/namelists.py index b6fb0f9..147a65b 100644 --- a/esm_runscripts/namelists.py +++ b/esm_runscripts/namelists.py @@ -266,10 +266,11 @@ def echam_determine_streams_from_nml(config): if "echam" in config["general"]["valid_model_names"]: nml = config["echam"]["namelists"]["namelist.echam"] mvstreams = nml["mvstreamctl"] - mvstreams_tags = [nml["filetag"] for nml in mvstreams] + mvstreams_tags = [nml.get("filetag") for nml in mvstreams] # NOTE(PG): There may still be warnings about missing files -- we # still need to implement an "allowed missing files" feature, but - # this should now put all the streams away correctly. + # this should now add all of the mvstreams that have a filetag to + # ECHAM's stream list. if not config["echam"].get("override_streams_from_namelist", False): config["echam"]["streams"] += mvstreams_tags else: @@ -277,6 +278,22 @@ def echam_determine_streams_from_nml(config): # restart will get messed up horribly. This just overrides # whatever was there in the default. It might be dangerous. config["echam"]["streams"] = mvstreams_tags + # As reference from the echam YAML: + # outdata_files: + # "[[streams-->STREAM]]": STREAM + # "[[streams-->STREAM]]_codes": STREAM_codes + # "[[streamsnc-->STREAM]]_nc": STREAM_nc + # + # outdata_sources: + # "[[streams-->STREAM]]": ${general.expid}_${start_date!syear}*.${start_date!sday}_STREAM + # "[[streams-->STREAM]]_codes": ${general.expid}_${start_date!syear}*.${start_date!sday}_STREAM.codes + # "[[streamsnc-->STREAM]]_nc": ${general.expid}_${start_date!syear!smonth}.${start_date!sday}_STREAM.nc + for stream in config['echam']['streams']: + config['echam']['outdata_files'][f'{stream}_codes'][stream] = f'{stream}'_codes + config['echam']['outdata_files'][f'{stream}_nc'][stream] = f'{stream}_nc' + config['echam']['outdata_sources'][f'{stream}'] = config['general']['expid']+"_*_"+stream + config['echam']['outdata_sources'][f'{stream}_codes'] = config['general']['expid']+"_*_"+stream+".codes" + config['echam']['outdata_sources'][f'{stream}_nc'] = config['general']['expid']+"_*_"+stream+".nc" return config @staticmethod From 526fc18567da28771b521abe05d8ead40c068fed Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Wed, 4 Aug 2021 17:48:15 +0200 Subject: [PATCH 09/11] fix(namelists): bad quote --- esm_runscripts/namelists.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esm_runscripts/namelists.py b/esm_runscripts/namelists.py index 147a65b..64ffe35 100644 --- a/esm_runscripts/namelists.py +++ b/esm_runscripts/namelists.py @@ -289,7 +289,7 @@ def echam_determine_streams_from_nml(config): # "[[streams-->STREAM]]_codes": ${general.expid}_${start_date!syear}*.${start_date!sday}_STREAM.codes # "[[streamsnc-->STREAM]]_nc": ${general.expid}_${start_date!syear!smonth}.${start_date!sday}_STREAM.nc for stream in config['echam']['streams']: - config['echam']['outdata_files'][f'{stream}_codes'][stream] = f'{stream}'_codes + config['echam']['outdata_files'][f'{stream}_codes'][stream] = f'{stream}_codes' config['echam']['outdata_files'][f'{stream}_nc'][stream] = f'{stream}_nc' config['echam']['outdata_sources'][f'{stream}'] = config['general']['expid']+"_*_"+stream config['echam']['outdata_sources'][f'{stream}_codes'] = config['general']['expid']+"_*_"+stream+".codes" From 59f5b84a4286a961e79532aa5e0f3dc6d7c530c9 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Wed, 4 Aug 2021 18:18:50 +0200 Subject: [PATCH 10/11] --wip-- [skip ci] --- esm_runscripts/namelists.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/esm_runscripts/namelists.py b/esm_runscripts/namelists.py index 64ffe35..1defcee 100644 --- a/esm_runscripts/namelists.py +++ b/esm_runscripts/namelists.py @@ -294,6 +294,13 @@ def echam_determine_streams_from_nml(config): config['echam']['outdata_sources'][f'{stream}'] = config['general']['expid']+"_*_"+stream config['echam']['outdata_sources'][f'{stream}_codes'] = config['general']['expid']+"_*_"+stream+".codes" config['echam']['outdata_sources'][f'{stream}_nc'] = config['general']['expid']+"_*_"+stream+".nc" + print("Hey bro, we need some debugging:") + print("We write the config as it is right now to the current working directory as >> stream_config.yaml <<") + import yaml + with open("stream_config.yaml", "w") as f: + yaml.dump(config, f) + print(config['echam']['outdata_files']) + print(config['echam']['outdata_sources']) return config @staticmethod From f0fecd92aaab53578de0cd61cd9d82d24a5521da Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Tue, 24 Aug 2021 15:39:24 +0200 Subject: [PATCH 11/11] feat: automatic echam streams from the namelist --- esm_runscripts/compute.py | 3 -- esm_runscripts/namelists.py | 42 ------------------- esm_runscripts/prepare.py | 82 +++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 45 deletions(-) diff --git a/esm_runscripts/compute.py b/esm_runscripts/compute.py index d20a16c..b087cd8 100644 --- a/esm_runscripts/compute.py +++ b/esm_runscripts/compute.py @@ -176,9 +176,6 @@ def modify_namelists(config): config[model] = Namelist.nmls_remove(config[model]) if model == "echam": config = Namelist.apply_echam_disturbance(config) - # NOTE(PG): This really doesn't belong in modify namelists, but, - # what the hell... - config = Namelist.echam_determine_streams_from_nml(config) config[model] = Namelist.nmls_modify(config[model]) config[model] = Namelist.nmls_finalize( config[model], config["general"]["verbose"] diff --git a/esm_runscripts/namelists.py b/esm_runscripts/namelists.py index 1defcee..3e0fe8a 100644 --- a/esm_runscripts/namelists.py +++ b/esm_runscripts/namelists.py @@ -261,48 +261,6 @@ def apply_echam_disturbance(config): ) return config - @staticmethod - def echam_determine_streams_from_nml(config): - if "echam" in config["general"]["valid_model_names"]: - nml = config["echam"]["namelists"]["namelist.echam"] - mvstreams = nml["mvstreamctl"] - mvstreams_tags = [nml.get("filetag") for nml in mvstreams] - # NOTE(PG): There may still be warnings about missing files -- we - # still need to implement an "allowed missing files" feature, but - # this should now add all of the mvstreams that have a filetag to - # ECHAM's stream list. - if not config["echam"].get("override_streams_from_namelist", False): - config["echam"]["streams"] += mvstreams_tags - else: - # NOTE(PG): I honestly am not sure if this will work, maybe the - # restart will get messed up horribly. This just overrides - # whatever was there in the default. It might be dangerous. - config["echam"]["streams"] = mvstreams_tags - # As reference from the echam YAML: - # outdata_files: - # "[[streams-->STREAM]]": STREAM - # "[[streams-->STREAM]]_codes": STREAM_codes - # "[[streamsnc-->STREAM]]_nc": STREAM_nc - # - # outdata_sources: - # "[[streams-->STREAM]]": ${general.expid}_${start_date!syear}*.${start_date!sday}_STREAM - # "[[streams-->STREAM]]_codes": ${general.expid}_${start_date!syear}*.${start_date!sday}_STREAM.codes - # "[[streamsnc-->STREAM]]_nc": ${general.expid}_${start_date!syear!smonth}.${start_date!sday}_STREAM.nc - for stream in config['echam']['streams']: - config['echam']['outdata_files'][f'{stream}_codes'][stream] = f'{stream}_codes' - config['echam']['outdata_files'][f'{stream}_nc'][stream] = f'{stream}_nc' - config['echam']['outdata_sources'][f'{stream}'] = config['general']['expid']+"_*_"+stream - config['echam']['outdata_sources'][f'{stream}_codes'] = config['general']['expid']+"_*_"+stream+".codes" - config['echam']['outdata_sources'][f'{stream}_nc'] = config['general']['expid']+"_*_"+stream+".nc" - print("Hey bro, we need some debugging:") - print("We write the config as it is right now to the current working directory as >> stream_config.yaml <<") - import yaml - with open("stream_config.yaml", "w") as f: - yaml.dump(config, f) - print(config['echam']['outdata_files']) - print(config['echam']['outdata_sources']) - return config - @staticmethod def nmls_finalize(mconfig, verbose): """ diff --git a/esm_runscripts/prepare.py b/esm_runscripts/prepare.py index 388d7b5..705f0d4 100644 --- a/esm_runscripts/prepare.py +++ b/esm_runscripts/prepare.py @@ -28,6 +28,88 @@ def mini_resolve_variable_date_file(date_file, config): return date_file +def maybe_add_extra_echam_streams(config): + import f90nml + import os + + if "echam" in config["general"]["valid_model_names"]: + mconfig = config["echam"] + if not mconfig.get("determine_echam_streams_from_namelist", False): + return config + + template_namelist_dir = mconfig["namelist_dir"] + + # Only solve my edge case for right now. Normally namelsist dir should + # be a full thing. + # + # This is horrible. Just horrible. + if "${project_base}" in template_namelist_dir: + template_namelist_dir = template_namelist_dir.replace("${project_base}", config["general"]["project_base"]) + + nml = f90nml.read(os.path.join(template_namelist_dir, "namelist.echam")) + mvstreams = nml.get("mvstreamctl") + # If only one mvstreamctl is there, it comes back as a namelist, but we + # want lists: + if not isinstance(mvstreams, list): + mvstreams = [mvstreams] + extra_streams = [] + for mvstream in mvstreams: + if mvstream is not None: + filetag = mvstream.get("filetag") + if filetag: + extra_streams.append(filetag) + else: + target = mvstream.get("target", "*m") # According to the handbook, the default is "*m" if not set + if target.startswith("*"): + ending = target[1:] + source = mvstream.get("source") # NOTE(PG): Are these ever lists?? + if isinstance(source, str): + extra_streams.append(source+ending) + elif isinstance(source, list): + sources = source # Pluralize the variable name to make it more obvious we have a list + for source in sources: + extra_streams.append(source+ending) + else: + extra_streams.append(target) + set_streams = nml.get("set_stream") + # If only one set_stream is there, it comes back as a namelist, but we + # want lists: + if not isinstance(set_streams, list): + set_streams = [set_streams] + for set_stream in set_streams: + if set_stream is not None: + if set_stream.get("lpost", 1): # 1 is the safer option. In the worst case, esm-tools complain about not being to move some files + if set_stream.get("post_suf"): + # User has defined their own suffix, we need to add that to the list: + extra_streams.append(set_stream.get("post_suf")) + else: + # User did not define a own suffix, but the default (if I understand the handbook) is to just use the stream name: + extra_streams.append(set_stream.get("stream")) + # Apparently you can also rename the restart files uniquely + if set_stream.get("rest_suf"): + extra_streams.append(set_stream.get("rest_suf")) + set_stream_elements = nml.get("set_stream_element") + # If only one set_stream_elements is there, it comes back as a namelist, but we + # want lists: + if not isinstance(set_stream_elements, list): + set_stream_elements = [set_stream_elements] + for set_stream_element in set_stream_elements: + if set_stream_element is not None: + lpost = set_stream_element.get("lpost", 1) + lrerun = set_stream_element.get("lrerun", 1) + stream = set_stream_element['stream'] # Will ECHAM crash if there is no stream in one of these chapter??? + if lpost or lrerun: + extra_streams.append(stream) + + # Remove duplicates: + extra_streams = list(set(extra_streams)) + for stream in extra_streams: + if stream not in config['echam']['streams']: + print(f"NOTE:\t\tAdding the following stream detected in namelist.echam which was not declared in echam.yaml\n \t\t{stream}") + config['echam']['streams'].append(stream) + return config + + def _read_date_file(config): import os import logging