diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 1aba3187d2..2a3f855486 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1194,7 +1194,8 @@ def make_module_extra_extensions(self): lines = [self.module_extra_extensions] # set environment variable that specifies list of extensions - exts_list = ','.join(['%s-%s' % (ext[0], ext[1]) for ext in self.cfg['exts_list']]) + # We need only name and version, so don't resolve templates + exts_list = ','.join(['-'.join(ext[:2]) for ext in self.cfg.get_ref('exts_list')]) env_var_name = convert_name(self.name, upper=True) lines.append(self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list)) @@ -1207,7 +1208,7 @@ def make_module_footer(self): footer = [self.module_generator.comment("Built with EasyBuild version %s" % VERBOSE_VERSION)] # add extra stuff for extensions (if any) - if self.cfg['exts_list']: + if self.cfg.get_ref('exts_list'): footer.append(self.make_module_extra_extensions()) # include modules footer if one is specified @@ -1791,7 +1792,7 @@ def fetch_step(self, skip_checksums=False): trace_msg(msg) # fetch extensions - if self.cfg['exts_list']: + if self.cfg.get_ref('exts_list'): self.exts = self.fetch_extension_sources(skip_checksums=skip_checksums) # create parent dirs in install and modules path already @@ -2063,7 +2064,7 @@ def extensions_step(self, fetch=False): - find source for extensions, in 'extensions' (and 'packages' for legacy reasons) - run extra_extensions """ - if len(self.cfg['exts_list']) == 0: + if not self.cfg.get_ref('exts_list'): self.log.debug("No extensions in exts_list") return diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 3faeb84b90..d79eed817b 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -45,6 +45,7 @@ import os import re from distutils.version import LooseVersion +from contextlib import contextmanager import easybuild.tools.filetools as filetools from easybuild.base import fancylogger @@ -383,6 +384,23 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): return toolchain_hierarchy +@contextmanager +def disable_templating(ec): + """Temporarily disable templating on the given EasyConfig + + Usage: + with disable_templating(ec): + # Do what you want without templating + # Templating set to previous value + """ + old_enable_templating = ec.enable_templating + ec.enable_templating = False + try: + yield old_enable_templating + finally: + ec.enable_templating = old_enable_templating + + class EasyConfig(object): """ Class which handles loading, reading, validation of easyconfigs @@ -592,18 +610,15 @@ def set_keys(self, params): """ # disable templating when setting easyconfig parameters # required to avoid problems with values that need more parsing to be done (e.g. dependencies) - prev_enable_templating = self.enable_templating - self.enable_templating = False - - for key in sorted(params.keys()): - # validations are skipped, just set in the config - if key in self._config.keys(): - self[key] = params[key] - self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) - else: - raise EasyBuildError("Unknown easyconfig parameter: %s (value '%s')", key, params[key]) - - self.enable_templating = prev_enable_templating + with disable_templating(self): + for key in sorted(params.keys()): + # validations are skipped, just set in the config + if key in self._config.keys(): + self[key] = params[key] + self.log.info("setting easyconfig parameter %s: value %s (type: %s)", + key, self[key], type(self[key])) + else: + raise EasyBuildError("Unknown easyconfig parameter: %s (value '%s')", key, params[key]) def parse(self): """ @@ -647,42 +662,39 @@ def parse(self): # templating is disabled when parse_hook is called to allow for easy updating of mutable easyconfig parameters # (see also comment in resolve_template) - prev_enable_templating = self.enable_templating - self.enable_templating = False - - # if any lists of dependency versions are specified over which we should iterate, - # deal with them now, before calling parse hook, parsing of dependencies & iterative easyconfig parameters... - self.handle_multi_deps() - - parse_hook_msg = None - if self.path: - parse_hook_msg = "Running %s hook for %s..." % (PARSE, os.path.basename(self.path)) - - # trigger parse hook - hooks = load_hooks(build_option('hooks')) - run_hook(PARSE, hooks, args=[self], msg=parse_hook_msg) - - # parse dependency specifications - # it's important that templating is still disabled at this stage! - self.log.info("Parsing dependency specifications...") - self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] - self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] - - # need to take into account that builddependencies may need to be iterated over, - # i.e. when the value is a list of lists of tuples - builddeps = self['builddependencies'] - if builddeps and all(isinstance(x, (list, tuple)) for b in builddeps for x in b): - self.iterate_options.append('builddependencies') - builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps] - else: - builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps] - self['builddependencies'] = builddeps + with disable_templating(self): + # if any lists of dependency versions are specified over which we should iterate, + # deal with them now, before calling parse hook, parsing of dependencies & iterative easyconfig parameters + self.handle_multi_deps() - # keep track of parsed multi deps, they'll come in handy during sanity check & module steps... - self.multi_deps = self.get_parsed_multi_deps() + parse_hook_msg = None + if self.path: + parse_hook_msg = "Running %s hook for %s..." % (PARSE, os.path.basename(self.path)) + + # trigger parse hook + hooks = load_hooks(build_option('hooks')) + run_hook(PARSE, hooks, args=[self], msg=parse_hook_msg) + + # parse dependency specifications + # it's important that templating is still disabled at this stage! + self.log.info("Parsing dependency specifications...") + self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] + self['hiddendependencies'] = [ + self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies'] + ] + + # need to take into account that builddependencies may need to be iterated over, + # i.e. when the value is a list of lists of tuples + builddeps = self['builddependencies'] + if builddeps and all(isinstance(x, (list, tuple)) for b in builddeps for x in b): + self.iterate_options.append('builddependencies') + builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps] + else: + builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps] + self['builddependencies'] = builddeps - # restore templating - self.enable_templating = prev_enable_templating + # keep track of parsed multi deps, they'll come in handy during sanity check & module steps... + self.multi_deps = self.get_parsed_multi_deps() # update templating dictionary self.generate_template_values() @@ -1108,63 +1120,57 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals :param always_overwrite: overwrite existing file at specified location without use of --force :param backup: create backup of existing file before overwriting it """ - orig_enable_templating = self.enable_templating - # templated values should be dumped unresolved - self.enable_templating = False - - # build dict of default values - default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG]) - default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options])) + with disable_templating(self): + # build dict of default values + default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG]) + default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options])) + + self.generate_template_values() + templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS]) + + # create reverse map of templates, to inject template values where possible + # longer template values are considered first, shorter template keys get preference over longer ones + sorted_keys = sorted(self.template_values, key=lambda k: (len(self.template_values[k]), -len(k)), + reverse=True) + templ_val = OrderedDict([]) + for key in sorted_keys: + # shortest template 'key' is retained in case of duplicates + # ('namelower' is preferred over 'github_account') + # only template values longer than 2 characters are retained + if self.template_values[key] not in templ_val and len(self.template_values[key]) > 2: + templ_val[self.template_values[key]] = key + + toolchain_hierarchy = None + if not explicit_toolchains: + try: + toolchain_hierarchy = get_toolchain_hierarchy(self['toolchain']) + except EasyBuildError as err: + # don't fail hard just because we can't get the hierarchy + self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump method, ' + 'error:\n%s', self['toolchain'], str(err)) - self.generate_template_values() - templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS]) - - # create reverse map of templates, to inject template values where possible - # longer template values are considered first, shorter template keys get preference over longer ones - sorted_keys = sorted(self.template_values, key=lambda k: (len(self.template_values[k]), -len(k)), reverse=True) - templ_val = OrderedDict([]) - for key in sorted_keys: - # shortest template 'key' is retained in case of duplicates ('namelower' is preferred over 'github_account') - # only template values longer than 2 characters are retained - if self.template_values[key] not in templ_val and len(self.template_values[key]) > 2: - templ_val[self.template_values[key]] = key - - toolchain_hierarchy = None - if not explicit_toolchains: try: - toolchain_hierarchy = get_toolchain_hierarchy(self['toolchain']) - except EasyBuildError as err: - # don't fail hard just because we can't get the hierarchy - self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump method, ' - 'error:\n%s', self['toolchain'], str(err)) + ectxt = self.parser.dump(self, default_values, templ_const, templ_val, + toolchain_hierarchy=toolchain_hierarchy) + except NotImplementedError as err: + raise NotImplementedError(err) - try: - ectxt = self.parser.dump(self, default_values, templ_const, templ_val, - toolchain_hierarchy=toolchain_hierarchy) - except NotImplementedError as err: - # need to restore enable_templating value in case this method is caught in a try/except block and ignored - # (the ability to dump is not a hard requirement for build success) - self.enable_templating = orig_enable_templating - raise NotImplementedError(err) + self.log.debug("Dumped easyconfig: %s", ectxt) - self.log.debug("Dumped easyconfig: %s", ectxt) + if build_option('dump_autopep8'): + autopep8_opts = { + 'aggressive': 1, # enable non-whitespace changes, but don't be too aggressive + 'max_line_length': 120, + } + self.log.info("Reformatting dumped easyconfig using autopep8 (options: %s)", autopep8_opts) + ectxt = autopep8.fix_code(ectxt, options=autopep8_opts) + self.log.debug("Dumped easyconfig after autopep8 reformatting: %s", ectxt) - if build_option('dump_autopep8'): - autopep8_opts = { - 'aggressive': 1, # enable non-whitespace changes, but don't be too aggressive - 'max_line_length': 120, - } - self.log.info("Reformatting dumped easyconfig using autopep8 (options: %s)", autopep8_opts) - ectxt = autopep8.fix_code(ectxt, options=autopep8_opts) - self.log.debug("Dumped easyconfig after autopep8 reformatting: %s", ectxt) + if not ectxt.endswith('\n'): + ectxt += '\n' - if not ectxt.endswith('\n'): - ectxt += '\n' - - write_file(fp, ectxt, always_overwrite=always_overwrite, backup=backup, verbose=backup) - - self.enable_templating = orig_enable_templating + write_file(fp, ectxt, always_overwrite=always_overwrite, backup=backup, verbose=backup) def _validate(self, attr, values): # private method """ @@ -1473,7 +1479,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False): # (true) boolean value simply indicates that a system toolchain is used elif isinstance(tc_spec, bool) and tc_spec: - tc = {'name': SYSTEM_TOOLCHAIN_NAME, 'version': ''} + tc = {'name': SYSTEM_TOOLCHAIN_NAME, 'version': ''} # two-element list/tuple value indicates custom toolchain specification elif isinstance(tc_spec, (list, tuple,)): @@ -1593,17 +1599,12 @@ def _generate_template_values(self, ignore=None): # step 1-3 work with easyconfig.templates constants # disable templating with creating dict with template values to avoid looping back to here via __getitem__ - prev_enable_templating = self.enable_templating - - self.enable_templating = False - - if self.template_values is None: - # if no template values are set yet, initiate with a minimal set of template values; - # this is important for easyconfig that use %(version_minor)s to define 'toolchain', - # which is a pretty weird use case, but fine... - self.template_values = template_constant_dict(self, ignore=ignore) - - self.enable_templating = prev_enable_templating + with disable_templating(self): + if self.template_values is None: + # if no template values are set yet, initiate with a minimal set of template values; + # this is important for easyconfig that use %(version_minor)s to define 'toolchain', + # which is a pretty weird use case, but fine... + self.template_values = template_constant_dict(self, ignore=ignore) # grab toolchain instance with templating support enabled, # which is important in case the Toolchain instance was not created yet @@ -1611,9 +1612,8 @@ def _generate_template_values(self, ignore=None): # get updated set of template values, now with toolchain instance # (which is used to define the %(mpi_cmd_prefix)s template) - self.enable_templating = False - template_values = template_constant_dict(self, ignore=ignore, toolchain=toolchain) - self.enable_templating = prev_enable_templating + with disable_templating(self): + template_values = template_constant_dict(self, ignore=ignore, toolchain=toolchain) # update the template_values dict self.template_values.update(template_values) @@ -1656,13 +1656,8 @@ def get_ref(self, key): # see also comments in resolve_template # temporarily disable templating - prev_enable_templating = self.enable_templating - self.enable_templating = False - - ref = self[key] - - # restore previous value for 'enable_templating' - self.enable_templating = prev_enable_templating + with disable_templating(self): + ref = self[key] return ref diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index b44d5759fe..90ba521ecd 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -127,14 +127,14 @@ def __init__(self, mself, ext, extra_params=None): # make sure they are merged into self.cfg so they can be queried; # unknown easyconfig parameters are ignored since self.options may include keys only there for extensions; # this allows to specify custom easyconfig parameters on a per-extension basis - for key in self.options: + for key, value in self.options.items(): if key in self.cfg: - self.cfg[key] = resolve_template(self.options[key], self.cfg.template_values) + self.cfg[key] = value self.log.debug("Customising known easyconfig parameter '%s' for extension %s/%s: %s", - key, name, version, self.cfg[key]) + key, name, version, value) else: self.log.debug("Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s", - key, name, version, self.options[key]) + key, name, version, value) self.sanity_check_fail_msgs = [] diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index e3859880e1..c001b809d0 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -524,7 +524,8 @@ def _generate_extension_list(self): """ Generate a string with a comma-separated list of extensions. """ - exts_list = self.app.cfg['exts_list'] + # We need only name and version, so don't resolve templates + exts_list = self.app.cfg.get_ref('exts_list') extensions = ', '.join(sorted(['-'.join(ext[:2]) for ext in exts_list], key=str.lower)) return extensions diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 0797e76de5..e0660cc96b 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -471,8 +471,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): update_build_specs={'version': new_version}, update_dep_versions=False) tweaked_ec = process_easyconfig(tweaked_spec)[0] - tweaked_dict = tweaked_ec['ec'].asdict() - extensions = tweaked_dict['exts_list'] + extensions = tweaked_ec['ec']['exts_list'] # check one extension with the same name exists and that the version has been updated hit_extension = 0 for extension in extensions: