diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 80cddf8819..cd0a760e37 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -599,19 +599,16 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): template_values = copy.deepcopy(self.cfg.template_values) template_values.update(template_constant_dict(ext_src)) - # resolve templates in extension options - ext_options = resolve_template(ext_options, template_values) - - source_urls = ext_options.get('source_urls', []) + source_urls = resolve_template(ext_options.get('source_urls', []), template_values) checksums = ext_options.get('checksums', []) - download_instructions = ext_options.get('download_instructions') + download_instructions = resolve_template(ext_options.get('download_instructions'), template_values) if ext_options.get('nosource', None): self.log.debug("No sources for extension %s, as indicated by 'nosource'", ext_name) elif ext_options.get('sources', None): - sources = ext_options['sources'] + sources = resolve_template(ext_options['sources'], template_values) # only a single source file is supported for extensions currently, # see https://github.com/easybuilders/easybuild-framework/issues/3463 @@ -648,17 +645,18 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): }) else: - # use default template for name of source file if none is specified - default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values) # if no sources are specified via 'sources', fall back to 'source_tmpl' src_fn = ext_options.get('source_tmpl') if src_fn is None: - src_fn = default_source_tmpl + # use default template for name of source file if none is specified + src_fn = '%(name)s-%(version)s.tar.gz' elif not isinstance(src_fn, string_type): error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s" raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn) + src_fn = resolve_template(src_fn, template_values) + if fetch_files: src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, force_download=force_download, @@ -689,7 +687,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): raise EasyBuildError('Checksum verification for extension source %s failed', src_fn) # locate extension patches (if any), and verify checksums - ext_patches = ext_options.get('patches', []) + ext_patches = resolve_template(ext_options.get('patches', []), template_values) if fetch_files: ext_patches = self.fetch_patches(patch_specs=ext_patches, extension=True) else: @@ -2879,7 +2877,7 @@ def extensions_step(self, fetch=False, install=True): fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods) - start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg['exts_list'])) + start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg.get_ref('exts_list'))) self.prepare_for_extensions() diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2b6aaa751a..7271f07a29 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -779,9 +779,13 @@ def count_files(self): """ Determine number of files (sources + patches) required for this easyconfig. """ - cnt = len(self['sources']) + len(self['patches']) - for ext in self['exts_list']: + # No need to resolve templates as we only need a count not the names + with self.disable_templating(): + cnt = len(self['sources']) + len(self['patches']) + exts = self['exts_list'] + + for ext in exts: if isinstance(ext, tuple) and len(ext) >= 3: ext_opts = ext[2] # check for 'sources' first, since that's also considered first by EasyBlock.fetch_extension_sources @@ -1641,8 +1645,9 @@ def _finalize_dependencies(self): filter_deps_specs = self.parse_filter_deps() for key in DEPENDENCY_PARAMETERS: - # loop over a *copy* of dependency dicts (with resolved templates); - deps = self[key] + # loop over a *copy* of dependency dicts with resolved templates, + # although some templates may not resolve yet (e.g. those relying on dependencies like %(pyver)s) + deps = resolve_template(self.get_ref(key), self.template_values, expect_resolved=False) # to update the original dep dict, we need to get a reference with templating disabled... deps_ref = self.get_ref(key) @@ -1814,11 +1819,14 @@ def get(self, key, default=None, resolve=True): # see also https://docs.python.org/2/reference/datamodel.html#object.__eq__ def __eq__(self, ec): """Is this EasyConfig instance equivalent to the provided one?""" - return self.asdict() == ec.asdict() + # Compare raw values to check that templates used are the same + with self.disable_templating(): + with ec.disable_templating(): + return self.asdict() == ec.asdict() def __ne__(self, ec): """Is this EasyConfig instance equivalent to the provided one?""" - return self.asdict() != ec.asdict() + return not self == ec def __hash__(self): """Return hash value for a hashable representation of this EasyConfig instance.""" @@ -1831,8 +1839,9 @@ def make_hashable(val): return val lst = [] - for (key, val) in sorted(self.asdict().items()): - lst.append((key, make_hashable(val))) + with self.disable_templating(): + for (key, val) in sorted(self.asdict().items()): + lst.append((key, make_hashable(val))) # a list is not hashable, but a tuple is return hash(tuple(lst)) @@ -1847,7 +1856,8 @@ def asdict(self): if self.enable_templating: if not self.template_values: self.generate_template_values() - value = resolve_template(value, self.template_values) + # Not all values can be resolved, e.g. %(installdir)s + value = resolve_template(value, self.template_values, expect_resolved=False) res[key] = value return res @@ -1997,10 +2007,11 @@ def get_module_path(name, generic=None, decode=True): return '.'.join(modpath + [module_name]) -def resolve_template(value, tmpl_dict): +def resolve_template(value, tmpl_dict, expect_resolved=True): """Given a value, try to susbstitute the templated strings with actual values. - value: some python object (supported are string, tuple/list, dict or some mix thereof) - tmpl_dict: template dictionary + - expect_resolved: Expects that all templates get resolved """ if isinstance(value, string_type): # simple escaping, making all '%foo', '%%foo', '%%%foo' post-templates values available, @@ -2032,7 +2043,13 @@ def resolve_template(value, tmpl_dict): try: value = value % tmpl_dict except KeyError: - _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict) + if expect_resolved: + depr_msg = ('Ignoring failure to resolve template value %s with dict %s.' % (value, tmpl_dict) + + '\n\tThis is deprecated and will lead to build failure. Check for correct escaping.') + if 'resolve-templates' in build_option('silence_deprecation_warnings'): + _log.warning(depr_msg, '5.0') + else: + _log.deprecated(depr_msg, '5.0') value = orig_value # Undo "%"-escaping else: # this block deals with references to objects and returns other references @@ -2047,11 +2064,12 @@ def resolve_template(value, tmpl_dict): # self._config['x']['y'] = z # it can not be intercepted with __setitem__ because the set is done at a deeper level if isinstance(value, list): - value = [resolve_template(val, tmpl_dict) for val in value] + value = [resolve_template(val, tmpl_dict, expect_resolved) for val in value] elif isinstance(value, tuple): - value = tuple(resolve_template(list(value), tmpl_dict)) + value = tuple(resolve_template(list(value), tmpl_dict, expect_resolved)) elif isinstance(value, dict): - value = {resolve_template(k, tmpl_dict): resolve_template(v, tmpl_dict) for k, v in value.items()} + value = {resolve_template(k, tmpl_dict, expect_resolved): resolve_template(v, tmpl_dict, expect_resolved) + for k, v in value.items()} return value diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 56f242ffce..125c68ba4b 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -129,7 +129,10 @@ def __init__(self, mself, ext, extra_params=None): self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) self.src_extract_cmd = self.ext.get('extract_cmd', None) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) - self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values) + # Some options may not be resolvable yet + self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), + self.cfg.template_values, + expect_resolved=False) if extra_params: self.cfg.extend_params(extra_params, overwrite=False) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 68ddba632a..ae6fffd6b7 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -340,7 +340,7 @@ def override_options(self): # override options descr = ("Override options", "Override default EasyBuild behavior.") - all_deprecations = ('python2', 'Lmod6', 'easyconfig', 'toolchain') + all_deprecations = ('python2', 'Lmod6', 'easyconfig', 'toolchain', 'resolve-templates') opts = OrderedDict({ 'accept-eula': ("Accept EULA for specified software [DEPRECATED, use --accept-eula-for instead!]", diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 19218eca6a..a502452b39 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1086,7 +1086,7 @@ def test_extension_source_tmpl(self): eb = EasyBlock(EasyConfig(self.eb_file)) error_pattern = r"source_tmpl value must be a string! " - error_pattern += r"\(found value of type 'list'\): \['bar-0\.0\.tar\.gz'\]" + error_pattern += r"\(found value of type 'list'\): \['%\(name\)s-%\(version\)s.tar.gz'\]" self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) self.contents = self.contents.replace("'source_tmpl': [SOURCE_TAR_GZ]", "'source_tmpl': SOURCE_TAR_GZ") diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 4b63dc605b..00acb3bd24 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3676,7 +3676,7 @@ def test_resolve_template(self): # On unknown values the value is returned unchanged for value in ('%(invalid)s', '%(name)s %(invalid)s', '%%%(invalid)s', '% %(invalid)s', '%s %(invalid)s'): - self.assertEqual(resolve_template(value, tmpl_dict), value) + self.assertEqual(resolve_template(value, tmpl_dict, expect_resolved=False), value) def test_det_subtoolchain_version(self): """Test det_subtoolchain_version function""" diff --git a/test/framework/easyconfigs/yeb/Python-2.7.10-intel-2018a.yeb b/test/framework/easyconfigs/yeb/Python-2.7.10-intel-2018a.yeb index 527f7bf8a5..e6cb14adcb 100644 --- a/test/framework/easyconfigs/yeb/Python-2.7.10-intel-2018a.yeb +++ b/test/framework/easyconfigs/yeb/Python-2.7.10-intel-2018a.yeb @@ -15,7 +15,7 @@ description: | toolchain: {name: intel, version: 2018a} toolchainopts: {pic: True, opt: True, optarch: True} -source_urls: ['http://www.python.org/ftp/python/%(version)s/'] +source_urls: ['http://www.python.org/ftp/%(namelower)s/%(version)s/'] sources: [*SOURCE_TGZ] # python needs bzip2 to build the bz2 package diff --git a/test/framework/easyconfigs/yeb/SQLite-3.8.10.2-foss-2018a.yeb b/test/framework/easyconfigs/yeb/SQLite-3.8.10.2-foss-2018a.yeb index a9d0afea39..acdfe5fa16 100644 --- a/test/framework/easyconfigs/yeb/SQLite-3.8.10.2-foss-2018a.yeb +++ b/test/framework/easyconfigs/yeb/SQLite-3.8.10.2-foss-2018a.yeb @@ -1,5 +1,5 @@ _internal_variables_: - - &versionstr 3081002 + - &versionstr '%(version_major)s081002' easyblock: ConfigureMake diff --git a/test/framework/yeb.py b/test/framework/yeb.py index 346ffa057b..caeeb73752 100644 --- a/test/framework/yeb.py +++ b/test/framework/yeb.py @@ -102,8 +102,8 @@ def test_parse_yeb(self): ec_eb = EasyConfig(ec_file) for key in sorted(ec_yeb.asdict()): - eb_val = ec_eb[key] - yeb_val = ec_yeb[key] + eb_val = ec_eb.get_ref(key) + yeb_val = ec_yeb.get_ref(key) if key == 'description': # multi-line string is always terminated with '\n' in YAML, so strip it off yeb_val = yeb_val.strip()