Modified ECHAM Streams based upon Namelist

esm_runscripts/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = 'dirk.barbi@awi.de'
-__version__ = "5.1.25"
+__version__ = "5.1.27"
 
 from .sim_objects import *
 from .batch_system import *

esm_runscripts/compute.py

@@ -284,7 +284,7 @@ def initialize_experiment_logfile(config):
 
         log_msg = f"# Beginning of Experiment {config['general']['expid']}"
         write_to_log(config, [log_msg], message_sep="")
-        
+
     write_to_log(
         config,
         [
@@ -302,7 +302,7 @@ def initialize_experiment_logfile(config):
             f"{config['general']['experiment_dir']}/log" \
             f"/{config['general']['expid']}_esm_runscripts_" \
             f"{config['general']['run_datestamp']}.log"
-        
+
         logger.trace_sink.def_path(logfile_path)
 
     return config
@@ -318,7 +318,7 @@ def _write_finalized_config(config):
     # here: https://pyyaml.org/wiki/PyYAMLDocumentation
     def date_representer(dumper, date):
         return dumper.represent_str(f"{date.output()}")
-        
+
     def calendar_representer(dumper, calendar):
         # Calendar has a __str__ method
         return dumper.represent_str(str(calendar))
@@ -338,29 +338,29 @@ class EsmConfigDumper(yaml.dumper.Dumper):
         pass
 
     # pyyaml does not support tuple and prints !!python/tuple
-    EsmConfigDumper.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list) 
+    EsmConfigDumper.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list)
 
     # Determine how non-built-in types will be printed be the YAML dumper
-    EsmConfigDumper.add_representer(esm_calendar.Date, date_representer) 
-    
-    EsmConfigDumper.add_representer(esm_calendar.esm_calendar.Calendar, 
-        calendar_representer) 
-        # yaml.representer.SafeRepresenter.represent_str) 
-        
-    EsmConfigDumper.add_representer(esm_parser.esm_parser.ConfigSetup, 
-        yaml.representer.SafeRepresenter.represent_dict) 
-        
+    EsmConfigDumper.add_representer(esm_calendar.Date, date_representer)
+
+    EsmConfigDumper.add_representer(esm_calendar.esm_calendar.Calendar,
+        calendar_representer)
+        # yaml.representer.SafeRepresenter.represent_str)
+
+    EsmConfigDumper.add_representer(esm_parser.esm_parser.ConfigSetup,
+        yaml.representer.SafeRepresenter.represent_dict)
+
     EsmConfigDumper.add_representer(batch_system, batch_system_representer)
 
     # format for the other ESM data structures
-    EsmConfigDumper.add_representer(esm_rcfile.esm_rcfile.EsmToolsDir, 
-        yaml.representer.SafeRepresenter.represent_str) 
-        
-    EsmConfigDumper.add_representer(esm_runscripts.coupler.coupler_class, 
-        coupler_representer) 
-        
-    EsmConfigDumper.add_representer(esm_runscripts.oasis.oasis, oasis_representer) 
-    
+    EsmConfigDumper.add_representer(esm_rcfile.esm_rcfile.EsmToolsDir,
+        yaml.representer.SafeRepresenter.represent_str)
+
+    EsmConfigDumper.add_representer(esm_runscripts.coupler.coupler_class,
+        coupler_representer)
+
+    EsmConfigDumper.add_representer(esm_runscripts.oasis.oasis, oasis_representer)
+
     config_file_path = \
         f"{config['general']['thisrun_config_dir']}"\
         f"/{config['general']['expid']}_finished_config.yaml"
@@ -369,11 +369,11 @@ class EsmConfigDumper(yaml.dumper.Dumper):
         config_final = copy.deepcopy(config) #PrevRunInfo
         del config_final["prev_run"]         #PrevRunInfo
 
-        out = yaml.dump(config_final, Dumper=EsmConfigDumper, width=10000, 
+        out = yaml.dump(config_final, Dumper=EsmConfigDumper, width=10000,
             indent=4)   #PrevRunInfo
         config_file.write(out)
     return config
-    
+
 
 def color_diff(diff):
     for line in diff:
@@ -417,7 +417,7 @@ def update_runscript(fromdir, scriptsdir, tfile, gconfig, file_type):
 
     # if `tfile` contains a full path of the runscript then remove the leading path
     tfile = os.path.basename(tfile)
-    
+
     # If the target file in ``scriptsdir`` does not exist, then copy the file
     # to the target.
     if not os.path.isfile(scriptsdir + "/" + tfile):
@@ -553,7 +553,7 @@ def copy_tools_to_thisrun(config):
     # protect such problems
     scriptsdir_deep_parents = list(pathlib.Path(scriptsdir).parents)[5:]
     deep_nesting_found = pathlib.Path(expdir) in scriptsdir_deep_parents
-    if deep_nesting_found: 
+    if deep_nesting_found:
         error_type = "runtime error"
         error_text = (
             f"deep recursion is detected in {__file__}:\n"
@@ -562,10 +562,10 @@ def copy_tools_to_thisrun(config):
             f"- experiment dir:     {expdir}"
         )
         # exit right away to prevent further recursion. There might still be
-        # running instances of esmr_runscripts and something like 
+        # running instances of esmr_runscripts and something like
         # `killall esm_runscripts` might be required
         esm_parser.user_error(error_type, error_text)
-    
+
     # If ``fromdir`` and ``scriptsdir`` are the same, this is already a computing
     # simulation which means we want to use the script in the experiment folder,
     # so no copying is needed
@@ -598,28 +598,28 @@ def copy_tools_to_thisrun(config):
         options_to_remove = [" -U ", " --update "]
         for option in options_to_remove:
             original_command = original_command.replace(option, " ")
-        
+
         # Before resubmitting the esm_runscripts, the path of the runscript
         # needs to be modified. Remove the absolute/relative path
         runscript_absdir, runscript = os.path.split(gconfig['runscript_abspath'])
         original_command_list = original_command.split()
         new_command_list = []
         for command in original_command_list:
             # current command will contain the full path, so replace it with
-            # the YAML file only since we are going to execute it from the 
+            # the YAML file only since we are going to execute it from the
             # `scriptsdir` now
             if runscript in command:
                 # gconfig['scriptname'] or `runscript` only contains the YAML file name
-                command = runscript 
+                command = runscript
             new_command_list.append(command)
 
         new_command = " ".join(new_command_list)
         restart_command = f"cd {scriptsdir}; esm_runscripts {new_command}"
-        
+
         # prevent continuous addition of --no-motd
         if not "--no-motd" in restart_command:
             restart_command += " --no-motd "
-            
+
         if config["general"]["verbose"]:
             print(restart_command)
         os.system(restart_command)
@@ -635,7 +635,7 @@ def _copy_preliminary_files_from_experiment_to_thisrun(config):
         f"{config['general']['expid']}_{config['general']['setup_name']}.date",
         "copy",
     )]
-    
+
     for filetype, filename, copy_or_link in filelist:
         source = config["general"]["experiment_" + filetype + "_dir"]
         dest = config["general"]["thisrun_" + filetype + "_dir"]
@@ -670,4 +670,4 @@ def _show_simulation_info(config):
     six.print_(80 * "=")
     six.print_()
     return config
-    
+

esm_runscripts/filelists.py

@@ -632,11 +632,13 @@ def copy_files(config, filetypes, source, target):
 
     for filetype in [filetype for filetype in filetypes if not filetype == "ignore"]:
         for model in config["general"]["valid_model_names"] + ["general"]:
-            movement_method = get_method(get_movement(config, model, filetype, source, target))
             if filetype + "_" + text_source in config[model]:
                 sourceblock = config[model][filetype + "_" + text_source]
                 targetblock = config[model][filetype + "_" + text_target]
                 for categ in sourceblock:
+                    movement_method = get_method(
+                        get_movement(config, model, categ, filetype, source, target)
+                    )
                     file_source = os.path.normpath(sourceblock[categ])
                     file_target = os.path.normpath(targetblock[categ])
                     if config["general"]["verbose"]:
@@ -801,6 +803,40 @@ def complete_all_file_movements(config):
                             config = complete_one_file_movement(config, model, filetype, movement, movement_type)
                         del mconfig["file_movements"][filetype]["all_directions"]
 
+            # Complete file specific movements with ``all_directions``
+            for file_in_fm in mconfig["file_movements"]:
+                # If it is a specific file, and not a file type
+                if file_in_fm not in (
+                    config["general"]["all_model_filetypes"]
+                    + ["scripts", "unknown"]
+                ):
+                    # Check syntax for restart files
+                    if (
+                        file_in_fm in mconfig.get("restart_in_files", {})
+                        or file_in_fm in mconfig.get("restart_out_files", {})
+                    ):
+                        esm_parser.user_error(
+                            "Movement direction not specified",
+                            f"'{model}.file_movements.{file_in_fm}' refers to a "
+                            + "restart file which can be moved/copied/link in two "
+                            + "directions, into the 'work' folder and out of the "
+                            + "'work' folder. Please, add the direction '_in' or "
+                            + f"'_out' to '{file_in_fm}':\n\n{model}:\n    "
+                            + f"file_movements:\n        {file_in_fm}_<in/out>:\n"
+                            + f"            [ ... ]"
+                        )
+                    # Solve ``all_directions``
+                    file_spec_movements = mconfig["file_movements"][file_in_fm]
+                    if "all_directions" in file_spec_movements:
+                        movement_type = file_spec_movements["all_directions"]
+                        for movement in [
+                            'init_to_exp', 'exp_to_run', 'run_to_work', 'work_to_run'
+                        ]:
+                            config = complete_one_file_movement(
+                                config, model, file_in_fm, movement, movement_type
+                            )
+                        del mconfig["file_movements"][file_in_fm]["all_directions"]
+
             if "default" in mconfig["file_movements"]:
                 if "all_directions" in mconfig["file_movements"]["default"]:
                     movement_type = mconfig["file_movements"]["default"]["all_directions"]
@@ -816,7 +852,20 @@ def complete_all_file_movements(config):
     return config
 
 
-def get_movement(config, model, filetype, source, target):
+def get_movement(config, model, categ, filetype, source, target):
+    # Remove globing strings from categ
+    if isinstance(categ, str):
+        categ = categ.split("_glob_")[0]
+    # Two type of directions are needed for restarts, therefore, the categories need an
+    # "_in" or "_out" at the end.
+    if filetype=="restart_in":
+        categ = f"{categ}_in"
+    elif filetype=="restart_out":
+        categ = f"{categ}_out"
+    # File specific movements
+    file_spec_movements = config[model]["file_movements"].get(categ, {})
+    # Movements associated to ``filetypes``
+    file_type_movements = config[model]["file_movements"][filetype]
     if source == "init":
         # Get the model-specific reusable_filetypes, if not existing, get the
         # general ones
@@ -825,13 +874,25 @@ def get_movement(config, model, filetype, source, target):
             config["general"]["reusable_filetypes"]
         )
         if config["general"]["run_number"] == 1 or filetype not in model_reusable_filetypes:
-            return config[model]["file_movements"][filetype]["init_to_exp"]
+            return file_spec_movements.get(
+                "init_to_exp",
+                file_type_movements["init_to_exp"]
+            )
         else:
-            return config[model]["file_movements"][filetype]["exp_to_run"]
+            return file_spec_movements.get(
+                "exp_to_run",
+                file_type_movements["exp_to_run"]
+            )
     elif source == "work":
-        return config[model]["file_movements"][filetype]["work_to_run"]
+        return file_spec_movements.get(
+            "work_to_run",
+            file_type_movements["work_to_run"]
+        )
     elif source == "thisrun" and target == "work":
-        return config[model]["file_movements"][filetype]["run_to_work"]
+        return file_spec_movements.get(
+            "run_to_work",
+            file_type_movements["run_to_work"]
+        )
     else:
         # This should NOT happen
         print(f"Error: Unknown file movement from {source} to {target}", flush=True)

esm_runscripts/namelists.py

@@ -149,7 +149,7 @@ def nmls_remove(mconfig):
                         del namelist_changes[namelist][change_chapter][key]
                         if remove_original_key:
                             del namelist_changes[namelist][change_chapter][original_key]
-                            
+
                         # mconfig instead of config, Grrrrr
                         print(f"- NOTE: removing the variable: {key} from the namelist: {namelist}")
 
@@ -322,7 +322,7 @@ def nmls_output(mconfig):
             print(f'::: end of the contents of {nml_name}\n')
         return mconfig
 
-    
+
     @staticmethod
     def nmls_output_all(config):
         six.print_(
@@ -342,6 +342,6 @@ def __init__(self, *args, **kwargs):
             DeprecationWarning,
             stacklevel=2,
         )
-    
+
 
         super(namelist, self).__init__(*args, **kwargs)