From 720eab3c098b7dc8f5d50b1a3d79c8d71123a99e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 27 Sep 2024 16:34:02 +0200 Subject: [PATCH 1/3] add support for injecting checksums for cargo crates Even though the `crates` are recognized as sources for `Cargo` easyconfigs the checksums are not written, at least when neither 'source_urls', 'sources' nor 'patches' are present. Handle `crates` like `sources` and add test- --- easybuild/framework/easyblock.py | 16 +++--- .../test_ecs/t/toy/toy-0.0-cargo.eb | 17 +++++++ test/framework/options.py | 49 ++++++++++++++++++ .../easybuild/easyblocks/generic/cargo.py | 51 +++++++++++++++++++ 4 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-cargo.eb create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/cargo.py diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 55107be6f9..85f906dc0a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -5573,8 +5573,8 @@ def make_checksum_lines(checksums, indent_level): if app.src: placeholder = '# PLACEHOLDER FOR SOURCES/PATCHES WITH CHECKSUMS' - # grab raw lines for source_urls, sources, data_sources, patches - keys = ['data_sources', 'patches', 'source_urls', 'sources'] + # grab raw lines for the following params + keys = ['data_sources', 'patches', 'source_urls', 'sources', 'crates'] raw = {} for key in keys: regex = re.compile(r'^(%s(?:.|\n)*?\])\s*$' % key, re.M) @@ -5585,15 +5585,11 @@ def make_checksum_lines(checksums, indent_level): _log.debug("Raw lines for %s easyconfig parameters: %s", '/'.join(keys), raw) - # inject combination of source_urls/sources/patches/checksums into easyconfig - # by replacing first occurence of placeholder that was put in place - sources_raw = raw.get('sources', '') - data_sources_raw = raw.get('data_sources', '') - source_urls_raw = raw.get('source_urls', '') - patches_raw = raw.get('patches', '') + # inject combination of the grabbed lines and the checksums into the easyconfig + # by replacing first the occurence of the placeholder that was put in place + raw_text = ''.join(raw.get(key, '') for key in keys) regex = re.compile(placeholder + '\n', re.M) - ectxt = regex.sub(source_urls_raw + sources_raw + data_sources_raw + patches_raw + checksums_txt + '\n', - ectxt, count=1) + ectxt = regex.sub(raw_text + checksums_txt + '\n', ectxt, count=1) # get rid of potential remaining placeholders ectxt = regex.sub('', ectxt) diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-cargo.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-cargo.eb new file mode 100644 index 0000000000..321b6e3d9f --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-cargo.eb @@ -0,0 +1,17 @@ +name = 'toy' +version = '0.0' +versionsuffix = '-cargo' + +easyblock = 'Cargo' + +homepage = 'https://easybuilders.github.io/easybuild' +description = "Toy C program, 100% toy." + +toolchain = SYSTEM + +crates = [ + ('toy', 'extra.txt'), + ('toy', '0.0_gzip.patch.gz'), +] + +moduleclass = 'tools' diff --git a/test/framework/options.py b/test/framework/options.py index 9775cb11aa..368950c024 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6392,6 +6392,55 @@ def test_inject_checksums(self): ] self.assertEqual(ext_opts['checksums'], expected_checksums) + # Also works for cargo crates + cargo_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-cargo.eb') + copy_file(cargo_ec, test_ec) + stdout, stderr = self._run_mock_eb([test_ec, '--inject-checksums'], raise_error=True, strip=True) + self.assertIn("injecting sha256 checksums in", stdout) + self.assertEqual(stderr, '') + expected_checksums = [ + {'toy-extra.txt': '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458'}, + {'toy-0.0_gzip.patch.gz': 'c5c51dd4b00fd490f8f8226f5fa609c30b66bda7ef6d3391ab2631508f3d5e41'}, + ] + patterns = [r"^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$"] + patterns.extend(r"^== \* %s: %s$" % next(iter(entry.items())) for entry in expected_checksums) + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + + ec = EasyConfigParser(test_ec).get_config_dict() + self.assertEqual(ec['checksums'], expected_checksums) + + # crates also work with sources and patches (unusual use case) + copy_file(cargo_ec, test_ec) + write_file(test_ec, textwrap.dedent(""" + sources = [SOURCE_TAR_GZ] + patches = [ + 'toy-0.0_fix-silly-typo-in-printf-statement.patch', + ] + """), append=True) + stdout, stderr = self._run_mock_eb([test_ec, '--inject-checksums'], raise_error=True, strip=True) + self.assertIn("injecting sha256 checksums in", stdout) + self.assertEqual(stderr, '') + expected_checksums = [ + # Main source + {'toy-0.0.tar.gz': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'}, + # Specified as "crates" + {'toy-extra.txt': '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458'}, + {'toy-0.0_gzip.patch.gz': 'c5c51dd4b00fd490f8f8226f5fa609c30b66bda7ef6d3391ab2631508f3d5e41'}, + # Patch + {'toy-0.0_fix-silly-typo-in-printf-statement.patch': + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'}, + ] + patterns = [r"^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$"] + patterns.extend(r"^== \* %s: %s$" % next(iter(entry.items())) for entry in expected_checksums) + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + + ec = EasyConfigParser(test_ec).get_config_dict() + self.assertEqual(ec['checksums'], expected_checksums) + # passing easyconfig filename as argument to --inject-checksums results in error being reported, # because it's not a valid type of checksum args = ['--inject-checksums', test_ec] diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/cargo.py b/test/framework/sandbox/easybuild/easyblocks/generic/cargo.py new file mode 100644 index 0000000000..403ba6725c --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/cargo.py @@ -0,0 +1,51 @@ +## +# Copyright 2009-2024 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig import CUSTOM + + +class Cargo(EasyBlock): + """Generic support for building/installing cargo crates.""" + + @staticmethod + def extra_options(): + """Custom easyconfig parameters for bar.""" + extra_vars = { + 'crates': [[], "List of (crate, version, [repo, rev]) tuples to use", CUSTOM], + } + return EasyBlock.extra_options(extra_vars) + + def __init__(self, *args, **kwargs): + """Constructor for Cargo easyblock.""" + super(Cargo, self).__init__(*args, **kwargs) + + # Populate sources from "crates" list of tuples + # For simplicity just assume (name,version.ext) tuples + sources = ['%s-%s' % crate_info for crate_info in self.cfg['crates']] + + # copy EasyConfig instance before we make changes to it + self.cfg = self.cfg.copy() + + self.cfg.update('sources', sources) From d793642765c631ddc3d24ed411b76eeee7a61fa6 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Sep 2024 09:41:01 +0200 Subject: [PATCH 2/3] Fix tests The added EC and EasyBlock requires changes to the expected output in some tests. --- test/framework/docs.py | 9 +++++++++ test/framework/filetools.py | 4 +++- test/framework/options.py | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index 865bdec716..2007b27af3 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -44,6 +44,7 @@ LIST_EASYBLOCKS_SIMPLE_TXT = """EasyBlock |-- bar |-- Bundle +|-- Cargo |-- CMakeMake |-- CmdCp |-- ConfigureMake @@ -107,6 +108,7 @@ LIST_EASYBLOCKS_DETAILED_TXT = """EasyBlock (easybuild.framework.easyblock) |-- bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py) |-- Bundle (easybuild.easyblocks.generic.bundle @ %(topdir)s/generic/bundle.py) +|-- Cargo (easybuild.easyblocks.generic.cargo @ %(topdir)s/generic/cargo.py) |-- CMakeMake (easybuild.easyblocks.generic.cmakemake @ %(topdir)s/generic/cmakemake.py) |-- CmdCp (easybuild.easyblocks.generic.cmdcp @ %(topdir)s/generic/cmdcp.py) |-- ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py) @@ -171,6 +173,7 @@ * bar * Bundle + * Cargo * CMakeMake * CmdCp * ConfigureMake @@ -262,6 +265,7 @@ * bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py) * Bundle (easybuild.easyblocks.generic.bundle @ %(topdir)s/generic/bundle.py) + * Cargo (easybuild.easyblocks.generic.cargo @ %(topdir)s/generic/cargo.py) * CMakeMake (easybuild.easyblocks.generic.cmakemake @ %(topdir)s/generic/cmakemake.py) * CmdCp (easybuild.easyblocks.generic.cmdcp @ %(topdir)s/generic/cmdcp.py) * ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py) @@ -352,6 +356,7 @@ LIST_EASYBLOCKS_SIMPLE_MD = """- **EasyBlock** - bar - Bundle + - Cargo - CMakeMake - CmdCp - ConfigureMake @@ -415,6 +420,7 @@ LIST_EASYBLOCKS_DETAILED_MD = """- **EasyBlock** (easybuild.framework.easyblock) - bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py) - Bundle (easybuild.easyblocks.generic.bundle @ %(topdir)s/generic/bundle.py) + - Cargo (easybuild.easyblocks.generic.cargo @ %(topdir)s/generic/cargo.py) - CMakeMake (easybuild.easyblocks.generic.cmakemake @ %(topdir)s/generic/cmakemake.py) - CmdCp (easybuild.easyblocks.generic.cmdcp @ %(topdir)s/generic/cmdcp.py) - ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py) @@ -725,6 +731,7 @@ def test_get_easyblock_classes(self): expected = [ 'Bundle', 'CMakeMake', + 'Cargo', 'ChildCustomDummyExtension', 'ChildDeprecatedDummyExtension', 'CmdCp', @@ -940,6 +947,7 @@ def test_list_software(self): 'homepage: https://easybuilders.github.io/easybuild', '', " * toy v0.0: gompi/2018a, system", + " * toy v0.0 (versionsuffix: '-cargo'): system", " * toy v0.0 (versionsuffix: '-deps'): system", " * toy v0.0 (versionsuffix: '-iter'): system", " * toy v0.0 (versionsuffix: '-multiple'): system", @@ -962,6 +970,7 @@ def test_list_software(self): 'version versionsuffix toolchain', '======= ============= ===========================', '``0.0`` ``gompi/2018a``, ``system``', + '``0.0`` ``-cargo`` ``system``', '``0.0`` ``-deps`` ``system``', '``0.0`` ``-iter`` ``system``', '``0.0`` ``-multiple`` ``system``', diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 35e17c9055..ed6ff1d5ec 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2698,11 +2698,13 @@ def test_index_functions(self): # load_index just returns None if there is no index in specified directory self.assertEqual(ft.load_index(self.test_prefix), None) + num_files = len(glob.glob(test_ecs + '/**/*.*', recursive=True)) + # create index for test easyconfigs; # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 104) + self.assertEqual(len(index), num_files) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), diff --git a/test/framework/options.py b/test/framework/options.py index 368950c024..e42306055a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1003,6 +1003,7 @@ def test_000_list_easyblocks(self): EasyBlock |-- bar |-- Bundle + |-- Cargo |-- CMakeMake |-- CmdCp |-- ConfigureMake From a90d9b7ee604f34cac703f59caf138a51117a10e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Oct 2024 12:45:41 +0200 Subject: [PATCH 3/3] Add customization point for easyblocks to specify additional parameters for "sources" Some easyblocks such as `Cargo` will use a custom EasyConfig parameters to fill the `sources` list. To avoid hard-coding them `inject_checksums` introduce a static method `src_parameter_names` to return a list of parameters that contribute to the final list of `sources` when parsing the EasyConfig. --- easybuild/framework/easyblock.py | 10 +++++++++- .../sandbox/easybuild/easyblocks/generic/cargo.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 85f906dc0a..f424cae1da 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -172,6 +172,14 @@ def extra_options(extra=None): return extra + @staticmethod + def src_parameter_names(): + """ + Return list of EasyConfig parameter that contribute to the sources in the `src` member + (or equivalently to the `sources` parameter of a parsed EasyConfig) + """ + return ['sources'] + # # INIT # @@ -5574,7 +5582,7 @@ def make_checksum_lines(checksums, indent_level): placeholder = '# PLACEHOLDER FOR SOURCES/PATCHES WITH CHECKSUMS' # grab raw lines for the following params - keys = ['data_sources', 'patches', 'source_urls', 'sources', 'crates'] + keys = ['data_sources', 'source_urls'] + app.src_parameter_names() + ['patches'] raw = {} for key in keys: regex = re.compile(r'^(%s(?:.|\n)*?\])\s*$' % key, re.M) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/cargo.py b/test/framework/sandbox/easybuild/easyblocks/generic/cargo.py index 403ba6725c..cb1622fd91 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/cargo.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/cargo.py @@ -37,6 +37,10 @@ def extra_options(): } return EasyBlock.extra_options(extra_vars) + @staticmethod + def src_parameter_names(): + return super(Cargo, Cargo).src_parameter_names() + ['crates'] + def __init__(self, *args, **kwargs): """Constructor for Cargo easyblock.""" super(Cargo, self).__init__(*args, **kwargs)