From aebcda04b996358b26bac8eadde1674416eb6227 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Jul 2021 10:24:45 +0200 Subject: [PATCH 01/10] Use skipTest than faulty C&P of return conditions --- test/framework/easyconfig.py | 105 ++++++++++++++--------------------- test/framework/github.py | 74 ++++++++---------------- test/framework/modules.py | 55 +++++++++--------- test/framework/options.py | 92 ++++++++++-------------------- test/framework/robot.py | 3 +- test/framework/style.py | 15 +---- test/framework/utilities.py | 36 ++++++++++++ 7 files changed, 161 insertions(+), 219 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 1f5067b804..9b3ea45ac3 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -80,12 +80,8 @@ from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import quote_str, quote_py_str from test.framework.github import GITHUB_TEST_ACCOUNT -from test.framework.utilities import find_full_path +from test.framework.utilities import find_full_path, requires_autopep8, requires_pycodestyle, requires_pygraph -try: - import pycodestyle # noqa # pylint:disable=unused-import -except ImportError: - pass EXPECTED_DOTTXT_TOY_DEPS = """digraph graphname { toy; @@ -2779,25 +2775,18 @@ def test_dump_order(self): 'foo_extra1', '', 'moduleclass', ''] self.assertEqual(param_regex.findall(ectxt), expected) + @requires_autopep8() def test_dump_autopep8(self): """Test dump() with autopep8 usage enabled (only if autopep8 is available).""" - try: - import autopep8 # noqa # pylint:disable=unused-import - except ImportError: - print("Skipping test_dump_autopep8, since autopep8 is not available") - return os.environ['EASYBUILD_DUMP_AUTOPEP8'] = '1' init_config() self.test_dump() del os.environ['EASYBUILD_DUMP_AUTOPEP8'] + @requires_pycodestyle() def test_dump_extra(self): """Test EasyConfig's dump() method for files containing extra values""" - if 'pycodestyle' not in sys.modules: - print("Skipping test_dump_extra pycodestyle is not available") - return - rawtxt = '\n'.join([ "easyblock = 'EB_foo'", '', @@ -2834,13 +2823,10 @@ def test_dump_extra(self): check_easyconfigs_style([testec]) + @requires_pycodestyle() def test_dump_template(self): """ Test EasyConfig's dump() method for files containing templates""" - if 'pycodestyle' not in sys.modules: - print("Skipping test_dump_template pycodestyle is not available") - return - rawtxt = '\n'.join([ "easyblock = 'EB_foo'", '', @@ -2923,13 +2909,10 @@ def test_dump_template(self): check_easyconfigs_style([testec]) + @requires_pycodestyle() def test_dump_comments(self): """ Test dump() method for files containing comments """ - if 'pycodestyle' not in sys.modules: - print("Skipping test_dump_comments pycodestyle is not available") - return - rawtxt = '\n'.join([ "# #", "# some header comment", @@ -3278,13 +3261,9 @@ def test_to_template_str(self): res = "sanity_check_paths = {\n 'files': [],\n 'dirs': ['lib/python%(pyshortver)s/site-packages'],\n}" self.assertEqual(to_template_str('sanity_check_paths', test_input, templ_const, templ_val), res) + @requires_pygraph() def test_dep_graph(self): """Test for dep_graph.""" - try: - import pygraph # noqa # pylint:disable=unused-import - except ImportError: - print("Skipping test_dep_graph, since pygraph is not available") - return test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') build_options = { @@ -3315,52 +3294,47 @@ def test_dep_graph(self): ordered_expected = '\n'.join(sorted(EXPECTED_DOTTXT_TOY_DEPS.split('\n'))) self.assertEqual(ordered_dottxt, ordered_expected) + @requires_pygraph() def test_dep_graph_multi_deps(self): """ Test for dep_graph using easyconfig that uses multi_deps. """ - try: - import pygraph # noqa # pylint:disable=unused-import - - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - build_options = { - 'external_modules_metadata': ConfigObj(), - 'valid_module_classes': module_classes(), - 'robot_path': [test_easyconfigs], - 'silent': True, - } - init_config(build_options=build_options) - - toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + build_options = { + 'external_modules_metadata': ConfigObj(), + 'valid_module_classes': module_classes(), + 'robot_path': [test_easyconfigs], + 'silent': True, + } + init_config(build_options=build_options) - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = toy_ec_txt + "\nmulti_deps = {'GCC': ['4.6.3', '4.8.3', '7.3.0-2.30']}" - write_file(test_ec, test_ec_txt) + toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') + toy_ec_txt = read_file(toy_ec) - ec_files = [(test_ec, False)] - ecs, _ = parse_easyconfigs(ec_files) + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = toy_ec_txt + "\nmulti_deps = {'GCC': ['4.6.3', '4.8.3', '7.3.0-2.30']}" + write_file(test_ec, test_ec_txt) - dot_file = os.path.join(self.test_prefix, 'test.dot') - ordered_ecs = resolve_dependencies(ecs, self.modtool, retain_all_deps=True) - dep_graph(dot_file, ordered_ecs) + ec_files = [(test_ec, False)] + ecs, _ = parse_easyconfigs(ec_files) - # hard check for expect .dot file contents - # 3 nodes should be there: 'GCC/6.4.0-2.28 (EXT)', 'toy', and 'intel/2018a' - # and 2 edges: 'toy -> intel' and 'toy -> "GCC/6.4.0-2.28 (EXT)"' - dottxt = read_file(dot_file) + dot_file = os.path.join(self.test_prefix, 'test.dot') + ordered_ecs = resolve_dependencies(ecs, self.modtool, retain_all_deps=True) + dep_graph(dot_file, ordered_ecs) - self.assertTrue(dottxt.startswith('digraph graphname {')) + # hard check for expect .dot file contents + # 3 nodes should be there: 'GCC/6.4.0-2.28 (EXT)', 'toy', and 'intel/2018a' + # and 2 edges: 'toy -> intel' and 'toy -> "GCC/6.4.0-2.28 (EXT)"' + dottxt = read_file(dot_file) - # just check for toy -> GCC deps - # don't bother doing full output check - # (different order for fields depending on Python version makes that tricky) - for gccver in ['4.6.3', '4.8.3', '7.3.0-2.30']: - self.assertTrue('"GCC/%s";' % gccver in dottxt) - self.assertTrue('"toy/0.0" -> "GCC/%s"' % gccver in dottxt) + self.assertTrue(dottxt.startswith('digraph graphname {')) - except ImportError: - print("Skipping test_dep_graph, since pygraph is not available") + # just check for toy -> GCC deps + # don't bother doing full output check + # (different order for fields depending on Python version makes that tricky) + for gccver in ['4.6.3', '4.8.3', '7.3.0-2.30']: + self.assertTrue('"GCC/%s";' % gccver in dottxt) + self.assertTrue('"toy/0.0" -> "GCC/%s"' % gccver in dottxt) def test_ActiveMNS_singleton(self): """Make sure ActiveMNS is a singleton class.""" @@ -4911,10 +4885,13 @@ def test_det_copy_ec_specs(self): self.assertEqual(paths, args[:-1]) self.assertEqual(target_path, args[-1]) + def test_det_copy_ec_specs_from_pr(self): + """Test det_copy_ec_specs function with --from-pr.""" + if self.skip_github_tests: - print("Skipping test_det_copy_ec_specs using --from-pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") + cwd = os.getcwd() # use fixed PR (speeds up the test due to caching in fetch_files_from_pr; # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22345 from_pr = 22345 diff --git a/test/framework/github.py b/test/framework/github.py index 6d88b90924..79de08d2de 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -201,8 +201,7 @@ def test_github_pick_default_branch(self): def test_github_walk(self): """test the gitubfs walk function""" if self.skip_github_tests: - print("Skipping test_walk, no GitHub token available?") - return + self.skipTest("No GitHub token available?") try: expected = [ @@ -217,8 +216,7 @@ def test_github_walk(self): def test_github_read_api(self): """Test the githubfs read function""" if self.skip_github_tests: - print("Skipping test_read_api, no GitHub token available?") - return + self.skipTest("No GitHub token available?") try: self.assertEqual(self.ghfs.read("a_directory/a_file.txt").strip(), b"this is a line of text") @@ -228,8 +226,7 @@ def test_github_read_api(self): def test_github_read(self): """Test the githubfs read function without using the api""" if self.skip_github_tests: - print("Skipping test_read, no GitHub token available?") - return + self.skipTest("No GitHub token available?") try: with self.mocked_stdout_stderr(): @@ -242,8 +239,7 @@ def test_github_read(self): def test_github_add_pr_labels(self): """Test add_pr_labels function.""" if self.skip_github_tests: - print("Skipping test_add_pr_labels, no GitHub token available?") - return + self.skipTest("No GitHub token available?") build_options = { 'pr_target_account': GITHUB_USER, @@ -280,8 +276,7 @@ def test_github_add_pr_labels(self): def test_github_fetch_pr_data(self): """Test fetch_pr_data function.""" if self.skip_github_tests: - print("Skipping test_fetch_pr_data, no GitHub token available?") - return + self.skipTest("No GitHub token available?") pr_data, _ = gh.fetch_pr_data(1, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT) @@ -302,8 +297,7 @@ def test_github_fetch_pr_data(self): def test_github_list_prs(self): """Test list_prs function.""" if self.skip_github_tests: - print("Skipping test_list_prs, no GitHub token available?") - return + self.skipTest("No GitHub token available?") parameters = ('closed', 'created', 'asc') @@ -324,8 +318,7 @@ def test_github_list_prs(self): def test_github_reasons_for_closing(self): """Test reasons_for_closing function.""" if self.skip_github_tests: - print("Skipping test_reasons_for_closing, no GitHub token available?") - return + self.skipTest("No GitHub token available?") repo_owner = gh.GITHUB_EB_MAIN repo_name = gh.GITHUB_EASYCONFIGS_REPO @@ -363,8 +356,7 @@ def test_github_reasons_for_closing(self): def test_github_close_pr(self): """Test close_pr function.""" if self.skip_github_tests: - print("Skipping test_close_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") build_options = { 'dry_run': True, @@ -408,8 +400,7 @@ def test_github_close_pr(self): def test_github_fetch_easyblocks_from_pr(self): """Test fetch_easyblocks_from_pr function.""" if self.skip_github_tests: - print("Skipping test_fetch_easyblocks_from_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, @@ -436,8 +427,7 @@ def test_github_fetch_easyblocks_from_pr(self): def test_github_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.skip_github_tests: - print("Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, @@ -474,8 +464,7 @@ def test_github_fetch_easyconfigs_from_pr(self): def test_github_fetch_files_from_pr_cache(self): """Test caching for fetch_files_from_pr.""" if self.skip_github_tests: - print("Skipping test_fetch_files_from_pr_cache, no GitHub token available?") - return + self.skipTest("No GitHub token available?") init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, @@ -591,8 +580,7 @@ def test_fetch_easyconfigs_from_commit(self): def test_github_fetch_latest_commit_sha(self): """Test fetch_latest_commit_sha function.""" if self.skip_github_tests: - print("Skipping test_fetch_latest_commit_sha, no GitHub token available?") - return + self.skipTest("No GitHub token available?") sha = gh.fetch_latest_commit_sha('easybuild-framework', 'easybuilders', github_user=GITHUB_TEST_ACCOUNT) self.assertTrue(re.match('^[0-9a-f]{40}$', sha)) @@ -603,8 +591,7 @@ def test_github_fetch_latest_commit_sha(self): def test_github_download_repo(self): """Test download_repo function.""" if self.skip_github_tests: - print("Skipping test_download_repo, no GitHub token available?") - return + self.skipTest("No GitHub token available?") cwd = os.getcwd() self.mock_stdout(True) @@ -672,15 +659,11 @@ def test_github_download_repo_commit(self): self.assertErrorRegex(EasyBuildError, "Failed to download tarball .* commit", gh.download_repo, path=self.test_prefix, commit='0000000000000000000000000000000000000000') + @unittest.skipIf(not HAVE_KEYRING, "keyring module not available") def test_install_github_token(self): """Test for install_github_token function.""" if self.skip_github_tests: - print("Skipping test_install_github_token, no GitHub token available?") - return - - if not HAVE_KEYRING: - print("Skipping test_install_github_token, keyring module not available") - return + self.skipTest("No GitHub token available?") random_user = ''.join(random.choice(ascii_letters) for _ in range(10)) self.assertEqual(gh.fetch_github_token(random_user), None) @@ -712,15 +695,11 @@ def fake_getpass(*args, **kwargs): self.assertTrue(token_installed) self.assertTrue(token == self.github_token) + @unittest.skipIf(not HAVE_KEYRING, "keyring module not available") def test_validate_github_token(self): """Test for validate_github_token function.""" if self.skip_github_tests: - print("Skipping test_validate_github_token, no GitHub token available?") - return - - if not HAVE_KEYRING: - print("Skipping test_validate_github_token, keyring module not available") - return + self.skipTest("No GitHub token available?") self.assertTrue(gh.validate_github_token(self.github_token, GITHUB_TEST_ACCOUNT)) @@ -737,8 +716,8 @@ def test_validate_github_token(self): def test_github_find_easybuild_easyconfig(self): """Test for find_easybuild_easyconfig function""" if self.skip_github_tests: - print("Skipping test_find_easybuild_easyconfig, no GitHub token available?") - return + self.skipTest("No GitHub token available?") + with self.mocked_stdout_stderr(): path = gh.find_easybuild_easyconfig(github_user=GITHUB_TEST_ACCOUNT) expected = os.path.join('e', 'EasyBuild', r'EasyBuild-[1-9]+\.[0-9]+\.[0-9]+\.eb') @@ -784,8 +763,7 @@ def test_github_det_commit_status(self): """Test det_commit_status function.""" if self.skip_github_tests: - print("Skipping test_det_commit_status, no GitHub token available?") - return + self.skipTest("No GitHub token available?") # ancient commit, from Jenkins era, no commit status available anymore commit_sha = 'ec5d6f7191676a86a18404616691796a352c5f1d' @@ -1087,8 +1065,7 @@ def test_github_det_patch_specs(self): def test_github_restclient(self): """Test use of RestClient.""" if self.skip_github_tests: - print("Skipping test_restclient, no GitHub token available?") - return + self.skipTest("No GitHub token available?") client = RestClient('https://api.github.com', username=GITHUB_TEST_ACCOUNT, token=self.github_token) @@ -1122,8 +1099,7 @@ def test_github_restclient(self): def test_github_create_delete_gist(self): """Test create_gist and delete_gist.""" if self.skip_github_tests: - print("Skipping test_restclient, no GitHub token available?") - return + self.skipTest("No GitHub token available?") test_txt = "This is just a test." @@ -1134,8 +1110,7 @@ def test_github_create_delete_gist(self): def test_github_det_account_repo_branch_for_pr(self): """Test det_account_branch_for_pr.""" if self.skip_github_tests: - print("Skipping test_det_account_branch_for_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") init_config(build_options={ 'pr_target_account': 'easybuilders', @@ -1254,8 +1229,7 @@ def test_push_branch_to_github(self): def test_github_pr_test_report(self): """Test for post_pr_test_report function.""" if self.skip_github_tests: - print("Skipping test_post_pr_test_report, no GitHub token available?") - return + self.skipTest("No GitHub token available?") init_config(build_options={ 'dry_run': True, diff --git a/test/framework/modules.py b/test/framework/modules.py index b69bb69c3e..502068476a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -850,43 +850,42 @@ def test_path_to_top_of_module_tree_hierarchical_mns(self): def test_path_to_top_of_module_tree_lua(self): """Test path_to_top_of_module_tree function on modules in Lua syntax.""" - if isinstance(self.modtool, Lmod): - orig_modulepath = os.environ.get('MODULEPATH') - self.modtool.unuse(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')) - curr_modulepath = os.environ.get('MODULEPATH') - error_msg = "Incorrect $MODULEPATH value after unuse: %s (orig: %s)" % (curr_modulepath, orig_modulepath) - self.assertEqual(curr_modulepath, None, error_msg) + if not isinstance(self.modtool, Lmod): + self.skipTest("Requires Lmod as modules tool") - top_moddir = os.path.join(self.test_prefix, 'test_modules') - core_dir = os.path.join(top_moddir, 'Core') - mkdir(core_dir, parents=True) - self.modtool.use(core_dir) - self.assertTrue(os.path.samefile(os.environ.get('MODULEPATH'), core_dir)) + orig_modulepath = os.environ.get('MODULEPATH') + self.modtool.unuse(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')) + curr_modulepath = os.environ.get('MODULEPATH') + error_msg = "Incorrect $MODULEPATH value after unuse: %s (orig: %s)" % (curr_modulepath, orig_modulepath) + self.assertEqual(curr_modulepath, None, error_msg) - # install toy modules in Lua syntax that are sufficient to test path_to_top_of_module_tree with - intel_mod_dir = os.path.join(top_moddir, 'Compiler', 'intel', '2016') - intel_mod = 'prepend_path("MODULEPATH", "%s")\n' % intel_mod_dir - write_file(os.path.join(core_dir, 'intel', '2016.lua'), intel_mod) + top_moddir = os.path.join(self.test_prefix, 'test_modules') + core_dir = os.path.join(top_moddir, 'Core') + mkdir(core_dir, parents=True) + self.modtool.use(core_dir) + self.assertTrue(os.path.samefile(os.environ.get('MODULEPATH'), core_dir)) - impi_mod_dir = os.path.join(top_moddir, 'MPI', 'intel', '2016', 'impi', '2016') - impi_mod = 'prepend_path("MODULEPATH", "%s")\n' % impi_mod_dir - write_file(os.path.join(intel_mod_dir, 'impi', '2016.lua'), impi_mod) + # install toy modules in Lua syntax that are sufficient to test path_to_top_of_module_tree with + intel_mod_dir = os.path.join(top_moddir, 'Compiler', 'intel', '2016') + intel_mod = 'prepend_path("MODULEPATH", "%s")\n' % intel_mod_dir + write_file(os.path.join(core_dir, 'intel', '2016.lua'), intel_mod) - imkl_mod = 'io.stderr:write("Hi from the imkl module")\n' - write_file(os.path.join(impi_mod_dir, 'imkl', '2016.lua'), imkl_mod) + impi_mod_dir = os.path.join(top_moddir, 'MPI', 'intel', '2016', 'impi', '2016') + impi_mod = 'prepend_path("MODULEPATH", "%s")\n' % impi_mod_dir + write_file(os.path.join(intel_mod_dir, 'impi', '2016.lua'), impi_mod) - self.assertEqual(self.modtool.available(), ['intel/2016']) + imkl_mod = 'io.stderr:write("Hi from the imkl module")\n' + write_file(os.path.join(impi_mod_dir, 'imkl', '2016.lua'), imkl_mod) - imkl_deps = ['intel/2016', 'impi/2016'] + self.assertEqual(self.modtool.available(), ['intel/2016']) - # modules that compose toolchain are expected to be loaded - self.modtool.load(imkl_deps) + imkl_deps = ['intel/2016', 'impi/2016'] - res = self.modtool.path_to_top_of_module_tree(core_dir, 'imkl/2016', impi_mod_dir, imkl_deps) - self.assertEqual(res, ['impi/2016', 'intel/2016']) + # modules that compose toolchain are expected to be loaded + self.modtool.load(imkl_deps) - else: - print("Skipping test_path_to_top_of_module_tree_lua, requires Lmod as modules tool") + res = self.modtool.path_to_top_of_module_tree(core_dir, 'imkl/2016', impi_mod_dir, imkl_deps) + self.assertEqual(res, ['impi/2016', 'intel/2016']) def test_interpret_raw_path_lua(self): """Test interpret_raw_path_lua method""" diff --git a/test/framework/options.py b/test/framework/options.py index 9727d3354c..2ca6de3840 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -66,15 +66,9 @@ from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import DARWIN, HAVE_ARCHSPEC, get_os_type from easybuild.tools.version import VERSION -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config, requires_pycodestyle from test.framework.github import ignore_rate_limit_in_pr -try: - import pycodestyle # noqa -except ImportError: - pass - - EXTERNAL_MODULES_METADATA = """[foobar/1.2.3] name = foo, bar version = 1.2.3, 3.2.1 @@ -1406,8 +1400,7 @@ def check_copied_files(): def test_github_copy_ec_from_pr(self): """Test combination of --copy-ec with --from-pr.""" if self.github_token is None: - print("Skipping test_copy_ec_from_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") test_working_dir = os.path.join(self.test_prefix, 'test_working_dir') mkdir(test_working_dir) @@ -2030,8 +2023,7 @@ def test_dry_run_categorized(self): def test_github_from_pr(self): """Test fetching easyconfigs from a PR.""" if self.github_token is None: - print("Skipping test_from_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -2099,8 +2091,7 @@ def test_github_from_pr(self): def test_github_from_pr_token_log(self): """Check that --from-pr doesn't leak GitHub token in log.""" if self.github_token is None: - print("Skipping test_from_pr_token_log, no GitHub token available?") - return + self.skipTest("No GitHub token available?") fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -2129,8 +2120,7 @@ def test_github_from_pr_token_log(self): def test_github_from_pr_listed_ecs(self): """Test --from-pr in combination with specifying easyconfigs on the command line.""" if self.github_token is None: - print("Skipping test_from_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -2182,8 +2172,7 @@ def test_github_from_pr_listed_ecs(self): def test_github_from_pr_x(self): """Test combination of --from-pr with --extended-dry-run.""" if self.github_token is None: - print("Skipping test_from_pr_x, no GitHub token available?") - return + self.skipTest("No GitHub token available?") fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -3869,8 +3858,7 @@ def test_xxx_include_generic_easyblocks(self): def test_github_xxx_include_easyblocks_from_pr(self): """Test --include-easyblocks-from-pr.""" if self.github_token is None: - print("Skipping test_preview_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") orig_local_sys_path = sys.path[:] @@ -4216,8 +4204,7 @@ def test_cleanup_tmpdir(self): def test_github_preview_pr(self): """Test --preview-pr.""" if self.github_token is None: - print("Skipping test_preview_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') eb_file = os.path.join(test_ecs_path, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb') @@ -4235,8 +4222,7 @@ def test_github_preview_pr(self): def test_github_review_pr(self): """Test --review-pr.""" if self.github_token is None: - print("Skipping test_review_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") # PR for bwidget 1.10.1 easyconfig, see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227 args = [ @@ -4487,8 +4473,7 @@ def _run_mock_eb(self, args, strip=False, **kwargs): def test_new_branch_github(self): """Test for --new-branch-github.""" if self.github_token is None: - print("Skipping test_create_branch_github, no GitHub token available?") - return + self.skipTest("No GitHub token available?") topdir = os.path.dirname(os.path.abspath(__file__)) @@ -4563,8 +4548,7 @@ def test_new_branch_github(self): def test_github_new_pr_from_branch(self): """Test --new-pr-from-branch.""" if self.github_token is None: - print("Skipping test_github_new_pr_from_branch, no GitHub token available?") - return + self.skipTest("No GitHub token available?") # see https://github.com/boegel/easybuild-easyconfigs/tree/test_new_pr_from_branch_DO_NOT_REMOVE # branch created specifically for this test, @@ -4603,8 +4587,7 @@ def test_github_new_pr_from_branch(self): def test_update_branch_github(self): """Test --update-branch-github.""" if self.github_token is None: - print("Skipping test_update_branch_github, no GitHub token available?") - return + self.skipTest("No GitHub token available?") topdir = os.path.dirname(os.path.abspath(__file__)) test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') @@ -4633,8 +4616,7 @@ def test_update_branch_github(self): def test_github_new_update_pr(self): """Test use of --new-pr (dry run only).""" if self.github_token is None: - print("Skipping test_new_update_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") # copy toy test easyconfig topdir = os.path.dirname(os.path.abspath(__file__)) @@ -4861,8 +4843,7 @@ def test_github_new_pr_warning_missing_patch(self): """Test warning printed by --new-pr (dry run only) when a specified patch file could not be found.""" if self.github_token is None: - print("Skipping test_new_pr_warning_missing_patch, no GitHub token available?") - return + self.skipTest("No GitHub token available?") topdir = os.path.dirname(os.path.abspath(__file__)) test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') @@ -4902,8 +4883,7 @@ def test_github_new_pr_warning_missing_patch(self): def test_github_sync_pr_with_develop(self): """Test use of --sync-pr-with-develop (dry run only).""" if self.github_token is None: - print("Skipping test_sync_pr_with_develop, no GitHub token available?") - return + self.skipTest("No GitHub token available?") # use https://github.com/easybuilders/easybuild-easyconfigs/pull/9150, # which is a PR from boegel:develop to easybuilders:develop @@ -4931,8 +4911,7 @@ def test_github_sync_pr_with_develop(self): def test_github_sync_branch_with_develop(self): """Test use of --sync-branch-with-develop (dry run only).""" if self.github_token is None: - print("Skipping test_sync_pr_with_develop, no GitHub token available?") - return + self.skipTest("No GitHub token available?") # see https://github.com/boegel/easybuild-easyconfigs/tree/test_new_pr_from_branch_DO_NOT_REMOVE test_branch = 'test_new_pr_from_branch_DO_NOT_REMOVE' @@ -4960,8 +4939,7 @@ def test_github_sync_branch_with_develop(self): def test_github_new_pr_python(self): """Check generated PR title for --new-pr on easyconfig that includes Python dependency.""" if self.github_token is None: - print("Skipping test_new_pr_python, no GitHub token available?") - return + self.skipTest("No GitHub token available?") # copy toy test easyconfig test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -5006,8 +4984,7 @@ def test_github_new_pr_delete(self): """Test use of --new-pr to delete easyconfigs.""" if self.github_token is None: - print("Skipping test_new_pr_delete, no GitHub token available?") - return + self.skipTest("No GitHub token available?") ec_name = 'bzip2-1.0.8.eb' args = [ @@ -5032,8 +5009,7 @@ def test_github_new_pr_dependencies(self): """Test use of --new-pr with automatic dependency lookup.""" if self.github_token is None: - print("Skipping test_new_pr_dependencies, no GitHub token available?") - return + self.skipTest("No GitHub token available?") foo_eb = '\n'.join([ 'easyblock = "ConfigureMake"', @@ -5082,8 +5058,7 @@ def test_github_new_pr_easyblock(self): """ if self.github_token is None: - print("Skipping test_new_pr_easyblock, no GitHub token available?") - return + self.skipTest("SNo GitHub token available?") topdir = os.path.dirname(os.path.abspath(__file__)) toy_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') @@ -5109,8 +5084,7 @@ def test_github_merge_pr(self): """ Test use of --merge-pr (dry run)""" if self.github_token is None: - print("Skipping test_merge_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") # start by making sure --merge-pr without dry-run errors out for a closed PR args = [ @@ -5215,8 +5189,7 @@ def test_github_merge_pr(self): def test_github_empty_pr(self): """Test use of --new-pr (dry run only) with no changes""" if self.github_token is None: - print("Skipping test_empty_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") # get file from develop branch full_url = URL_SEPARATOR.join([GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, @@ -5673,13 +5646,12 @@ def test_zip_logs(self): def test_debug_lmod(self): """Test use of --debug-lmod.""" - if isinstance(self.modtool, Lmod): - init_config(build_options={'debug_lmod': True}) - out = self.modtool.run_module('avail', return_output=True) + if not isinstance(self.modtool, Lmod): + self.skipTest("requires Lmod as modules tool") + init_config(build_options={'debug_lmod': True}) + out = self.modtool.run_module('avail', return_output=True) - self.assert_multi_regex([r"^Lmod version", r"^lmod\(--terse -D avail\)\{", ":avail"], out) - else: - print("Skipping test_debug_lmod, requires Lmod as modules tool") + self.assert_multi_regex([r"^Lmod version", r"^lmod\(--terse -D avail\)\{", ":avail"], out) def test_use_color(self): """Test use_color function.""" @@ -5879,12 +5851,9 @@ def test_parse_optarch(self): options.postprocess() self.assertEqual(options.options.optarch, optarch_parsed) + @requires_pycodestyle() def test_check_contrib_style(self): """Test style checks performed by --check-contrib + dedicated --check-style option.""" - if 'pycodestyle' not in sys.modules: - print("Skipping test_check_contrib_style pycodestyle is not available") - return - regex = re.compile(r"Running style check on 2 easyconfig\(s\)(.|\n)*>> All style checks PASSed!", re.M) args = [ '--check-style', @@ -5930,13 +5899,10 @@ def test_check_contrib_style(self): ] self.assert_multi_regex(patterns, stdout) + @requires_pycodestyle() def test_check_contrib_non_style(self): """Test non-style checks performed by --check-contrib.""" - if 'pycodestyle' not in sys.modules: - print("Skipping test_check_contrib_non_style pycodestyle is not available") - return - args = [ '--check-contrib', 'toy-0.0.eb', diff --git a/test/framework/robot.py b/test/framework/robot.py index 895c92f834..fbbc43ec64 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -756,8 +756,7 @@ def test_github_det_easyconfig_paths_from_commit(self): def test_github_det_easyconfig_paths_from_pr(self): """Test det_easyconfig_paths function, with --from-pr enabled as well.""" if self.github_token is None: - print("Skipping test_from_pr, no GitHub token available?") - return + self.skipTest("No GitHub token available?") fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) diff --git a/test/framework/style.py b/test/framework/style.py index 92f588cf3b..81040617b8 100644 --- a/test/framework/style.py +++ b/test/framework/style.py @@ -31,26 +31,19 @@ import glob import os import sys -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, requires_pycodestyle from unittest import TextTestRunner from easybuild.base import fancylogger from easybuild.framework.easyconfig.style import _eb_check_trailing_whitespace, check_easyconfigs_style -try: - import pycodestyle # noqa -except ImportError: - pass - class StyleTest(EnhancedTestCase): log = fancylogger.getLogger("StyleTest", fname=False) + @requires_pycodestyle() def test_style_conformance(self): """Check the easyconfigs for style""" - if 'pycodestyle' not in sys.modules: - print("Skipping test_style_conformance pycodestyle is not available") - return # all available easyconfig files test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -61,11 +54,9 @@ def test_style_conformance(self): self.assertEqual(result, 0, "No code style errors (and/or warnings) found.") + @requires_pycodestyle() def test_check_trailing_whitespace(self): """Test for trailing whitespace check.""" - if 'pycodestyle' not in sys.modules: - print("Skipping test_check_trailing_whitespace pycodestyle is not available") - return lines = [ "name = 'foo'", # no trailing whitespace diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 1807085d75..3c4654c80a 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -555,3 +555,39 @@ def find_full_path(base_path, trim=(lambda x: x)): break return full_path + + +def requires_pycodestyle(): + try: + import pycodestyle # noqa + ok = True + except ImportError: + ok = False + return unittest.skipUnless(ok, "no pycodestyle available") + + +def requires_autopep8(): + try: + import autopep8 # noqa + ok = True + except ImportError: + ok = False + return unittest.skipUnless(ok, "autopep8 is not available") + + +def requires_pygraph(): + try: + import pygraph # noqa + ok = True + except ImportError: + ok = False + return unittest.skipUnless(ok, "pygraph is not available") + + +def requires_PyYAML(): + try: + import yaml # noqa + ok = True + except ImportError: + ok = False + return unittest.skipUnless(ok, "PyYAML is not available") From d9c4b008c64d183cebf9de8f4f5627e860a3b0e6 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 1 Feb 2023 09:44:07 +0100 Subject: [PATCH 02/10] Silently skip SVN test on CI The pip-installable pysvn is incomplete and the full one is not easy to install. --- .github/workflows/unit_tests.yml | 1 - test/framework/repository.py | 9 ++------- test/framework/utilities.py | 20 ++++++++++++++++++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index bd2371f3b6..fae6ab3432 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -244,7 +244,6 @@ jobs: python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) IGNORE_PATTERNS="no GitHub token available" - IGNORE_PATTERNS+="|skipping SvnRepository test" IGNORE_PATTERNS+="|requires Lmod as modules tool" IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device" IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.7" diff --git a/test/framework/repository.py b/test/framework/repository.py index 37f0d9a729..84f12aa5f2 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -44,6 +44,7 @@ from easybuild.tools.repository.repository import init_repository from easybuild.tools.run import run_shell_cmd from easybuild.tools.version import VERSION +from test.framework.utilities import requires_pysvn class RepositoryTest(EnhancedTestCase): @@ -113,15 +114,9 @@ def test_gitrepo(self): shutil.rmtree(repo.wc) shutil.rmtree(tmpdir) + @requires_pysvn() def test_svnrepo(self): """Test using SvnRepository.""" - # only run this test if pysvn Python module is available - try: - from pysvn import ClientError # noqa - except ImportError: - print("(skipping SvnRepository test)") - return - # GitHub also supports SVN test_repo_url = 'https://github.com/easybuilders/testrepository' diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 3c4654c80a..351a9e4806 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -30,6 +30,7 @@ """ import copy import fileinput +import functools import os import re import shutil @@ -584,6 +585,25 @@ def requires_pygraph(): return unittest.skipUnless(ok, "pygraph is not available") +def requires_pysvn(): + try: + from pysvn import ClientError # noqa + ok = True + except ImportError: + ok = False + if 'CI' in os.environ: + # For CI skip silently, not easy enough to install, + # see https://github.com/leafvmaple/pysvn/issues/1 + def decorator(test_item): + @functools.wraps(test_item) + def skip_wrapper(*args, **kwargs): + return + return skip_wrapper + return decorator + else: + return unittest.skipUnless(ok, "PySVN is not available, use e.g. apt-get install python3-svn") + + def requires_PyYAML(): try: import yaml # noqa From 601e3ae80ab75ca5ba43fcea445df619f4a92869 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 1 Feb 2023 09:51:23 +0100 Subject: [PATCH 03/10] Avoid output of expected skip of LMod tests Simply return if `$TEST_EASYBUILD_MODULES_TOOL` is not set to Lmod. This avoids any "skip" output on CI --- .github/workflows/unit_tests.yml | 1 - test/framework/modules.py | 2 ++ test/framework/options.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index fae6ab3432..05b0a1d7ba 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -244,7 +244,6 @@ jobs: python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) IGNORE_PATTERNS="no GitHub token available" - IGNORE_PATTERNS+="|requires Lmod as modules tool" IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device" IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.7" IGNORE_PATTERNS+="|from cryptography.* import " diff --git a/test/framework/modules.py b/test/framework/modules.py index 502068476a..eb3743f90b 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -851,6 +851,8 @@ def test_path_to_top_of_module_tree_hierarchical_mns(self): def test_path_to_top_of_module_tree_lua(self): """Test path_to_top_of_module_tree function on modules in Lua syntax.""" if not isinstance(self.modtool, Lmod): + if os.environ.get('TEST_EASYBUILD_MODULES_TOOL') != 'Lmod': + return # Treat as success as nothing to do self.skipTest("Requires Lmod as modules tool") orig_modulepath = os.environ.get('MODULEPATH') diff --git a/test/framework/options.py b/test/framework/options.py index 2ca6de3840..22472b3db5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5647,6 +5647,8 @@ def test_zip_logs(self): def test_debug_lmod(self): """Test use of --debug-lmod.""" if not isinstance(self.modtool, Lmod): + if os.environ.get('TEST_EASYBUILD_MODULES_TOOL') != 'Lmod': + return # Treat as success as nothing to do self.skipTest("requires Lmod as modules tool") init_config(build_options={'debug_lmod': True}) out = self.modtool.run_module('avail', return_output=True) From 18f96c97c5dc39a0acbedbed6186cb9dbbcf9f74 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 1 Feb 2023 12:14:05 +0100 Subject: [PATCH 04/10] Move GC3Pie check to skip decorator --- test/framework/parallelbuild.py | 9 ++------- test/framework/utilities.py | 9 +++++++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 0c821dd882..1efbcf6d06 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -44,7 +44,7 @@ from easybuild.tools.options import parse_options from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel, submit_jobs from easybuild.tools.robot import resolve_dependencies - +from test.framework.utilities import requires_GC3Pie # test GC3Pie configuration with large resource specs GC3PIE_LOCAL_CONFIGURATION = """[resource/ebtestlocalhost] @@ -215,14 +215,9 @@ def test_build_easyconfigs_in_parallel_pbs_python(self): pbs_python.PbsJob = pbs_python_PbsJob self.mock_stdout(False) + @requires_GC3Pie() def test_build_easyconfigs_in_parallel_gc3pie(self): """Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job.""" - try: - import gc3libs # noqa (ignore unused import) - except ImportError: - print("GC3Pie not available, skipping test") - return - self.allow_deprecated_behaviour() # put GC3Pie config in place to use local host and fork/exec diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 351a9e4806..31f3515c68 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -576,6 +576,15 @@ def requires_autopep8(): return unittest.skipUnless(ok, "autopep8 is not available") +def requires_GC3Pie(): + try: + import gc3libs # noqa + ok = True + except ImportError: + ok = False + return unittest.skipUnless(ok, "GC3Pie not available") + + def requires_pygraph(): try: import pygraph # noqa From 69003de289711236e1c5492ff3cc3c9c3e1c0388 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 1 Feb 2023 12:57:52 +0100 Subject: [PATCH 05/10] Introduce skip marker for github token-using tests For PRs silently skip the tests when no token is available else use a skip. On GHA force enable the tests when the source repo is 'easybuilds' so a token is expected to be available. --- .github/workflows/unit_tests.yml | 10 +++ test/framework/easyconfig.py | 10 +-- test/framework/github.py | 107 ++++++++++++------------------- test/framework/options.py | 106 ++++++++---------------------- test/framework/robot.py | 7 +- 5 files changed, 83 insertions(+), 157 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 05b0a1d7ba..55850600a8 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -240,6 +240,16 @@ jobs: # create file owned by root but writable by anyone (used by test_copy_file) sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + + if [ "$GITHUB_REPOSITORY_OWNER" == 'easybuilders' ] || [ '${{github.event.pull_request.head.repo.owner}}' == 'easybuilders' ]; then + FORCE_EB_GITHUB_TESTS=1 + else + echo "Not force-enabling Github tests" + echo "\$GITHUB_EVENT_NAME=GITHUB_EVENT_NAME" + echo "\$GITHUB_REPOSITORY_OWNER=$GITHUB_REPOSITORY_OWNER" + echo "PR HEAD repo owner=${{github.event.pull_request.head.repo.owner}}" + fi + # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 9b3ea45ac3..fd4728c011 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -45,7 +45,6 @@ import easybuild.tools.build_log import easybuild.framework.easyconfig as easyconfig -import easybuild.tools.github as gh import easybuild.tools.systemtools as st from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER @@ -79,7 +78,7 @@ from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import quote_str, quote_py_str -from test.framework.github import GITHUB_TEST_ACCOUNT +from test.framework.github import requires_github_token from test.framework.utilities import find_full_path, requires_autopep8, requires_pycodestyle, requires_pygraph @@ -108,9 +107,6 @@ def setUp(self): if os.path.exists(self.eb_file): os.remove(self.eb_file) - github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) - self.skip_github_tests = github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None - self.orig_easyconfig_DEPRECATED_EASYCONFIG_PARAMETERS = easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS self.orig_easyconfig_DEPRECATED_EASYCONFIG_TEMPLATES = easyconfig.easyconfig.DEPRECATED_EASYCONFIG_TEMPLATES self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_PARAMETERS = easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS @@ -4885,12 +4881,10 @@ def test_det_copy_ec_specs(self): self.assertEqual(paths, args[:-1]) self.assertEqual(target_path, args[-1]) + @requires_github_token() def test_det_copy_ec_specs_from_pr(self): """Test det_copy_ec_specs function with --from-pr.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - cwd = os.getcwd() # use fixed PR (speeds up the test due to caching in fetch_files_from_pr; # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22345 diff --git a/test/framework/github.py b/test/framework/github.py index 79de08d2de..c66faf303c 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -100,6 +100,25 @@ def skip_wrapper(self, *args, **kwargs): return skip_wrapper +def requires_github_token(): + """Require a github token to be available unless $FORCE_EB_GITHUB_TESTS is set""" + if 'FORCE_EB_GITHUB_TESTS' in os.environ: + return unittest.skipIf(False, None) + github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) + if os.getenv('GITHUB_EVENT_NAME') != 'pull_request': + return unittest.skipUnless(github_token, "No GitHub token available") + elif github_token: + return unittest.skipIf(False, None) + else: + # For pull requests silently skip if no token is available as that is expected + def decorator(test_item): + @functools.wraps(test_item) + def skip_wrapper(*args, **kwargs): + return + return skip_wrapper + return decorator + + class GithubTest(EnhancedTestCase): """ small test for The github package This should not be to much, since there is an hourly limit of request @@ -118,8 +137,6 @@ def setUp(self): self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, username, None, token) - self.skip_github_tests = self.github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None - self.orig_testing_create_gist = easybuild.tools.testing.create_gist def tearDown(self): @@ -198,11 +215,9 @@ def test_github_pick_default_branch(self): self.assertEqual(pick_default_branch('easybuilders'), 'main') self.assertEqual(pick_default_branch('foobar'), 'master') + @requires_github_token() def test_github_walk(self): """test the gitubfs walk function""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - try: expected = [ (None, ['a_directory', 'second_dir'], ['README.md']), @@ -213,21 +228,17 @@ def test_github_walk(self): except IOError: pass + @requires_github_token() def test_github_read_api(self): """Test the githubfs read function""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - try: self.assertEqual(self.ghfs.read("a_directory/a_file.txt").strip(), b"this is a line of text") except IOError: pass + @requires_github_token() def test_github_read(self): """Test the githubfs read function without using the api""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - try: with self.mocked_stdout_stderr(): fp = self.ghfs.read("a_directory/a_file.txt", api=False) @@ -236,11 +247,9 @@ def test_github_read(self): except (IOError, OSError): pass + @requires_github_token() def test_github_add_pr_labels(self): """Test add_pr_labels function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - build_options = { 'pr_target_account': GITHUB_USER, 'pr_target_repo': GITHUB_EASYBLOCKS_REPO, @@ -273,11 +282,9 @@ def test_github_add_pr_labels(self): self.mock_stderr(False) self.assertIn("Could not determine any missing labels for PR #22088", stdout) + @requires_github_token() def test_github_fetch_pr_data(self): """Test fetch_pr_data function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - pr_data, _ = gh.fetch_pr_data(1, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT) self.assertEqual(pr_data['number'], 1) @@ -294,11 +301,9 @@ def test_github_fetch_pr_data(self): self.assertEqual(pr_data['reviews'][0]['user']['login'], 'boegel') self.assertEqual(pr_data['status_last_commit'], None) + @requires_github_token() def test_github_list_prs(self): """Test list_prs function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - parameters = ('closed', 'created', 'asc') init_config(build_options={'pr_target_account': GITHUB_USER, @@ -315,11 +320,9 @@ def test_github_list_prs(self): self.assertEqual(expected, output) + @requires_github_token() def test_github_reasons_for_closing(self): """Test reasons_for_closing function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - repo_owner = gh.GITHUB_EB_MAIN repo_name = gh.GITHUB_EASYCONFIGS_REPO @@ -353,11 +356,9 @@ def test_github_reasons_for_closing(self): for pattern in patterns: self.assertIn(pattern, stdout) + @requires_github_token() def test_github_close_pr(self): """Test close_pr function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - build_options = { 'dry_run': True, 'github_user': GITHUB_TEST_ACCOUNT, @@ -397,11 +398,9 @@ def test_github_close_pr(self): for pattern in patterns: self.assertIn(pattern, stdout) + @requires_github_token() def test_github_fetch_easyblocks_from_pr(self): """Test fetch_easyblocks_from_pr function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, }) @@ -424,11 +423,9 @@ def test_github_fetch_easyblocks_from_pr(self): except URLError as err: print("Ignoring URLError '%s' in test_fetch_easyblocks_from_pr" % err) + @requires_github_token() def test_github_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, }) @@ -461,11 +458,9 @@ def test_github_fetch_easyconfigs_from_pr(self): except URLError as err: print("Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err) + @requires_github_token() def test_github_fetch_files_from_pr_cache(self): """Test caching for fetch_files_from_pr.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, }) @@ -577,22 +572,18 @@ def test_fetch_easyconfigs_from_commit(self): self.assertEqual(len(res), 1) self.assertIn("v4.9.0 (30 December 2023)", read_file(res[0])) + @requires_github_token() def test_github_fetch_latest_commit_sha(self): """Test fetch_latest_commit_sha function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - sha = gh.fetch_latest_commit_sha('easybuild-framework', 'easybuilders', github_user=GITHUB_TEST_ACCOUNT) self.assertTrue(re.match('^[0-9a-f]{40}$', sha)) sha = gh.fetch_latest_commit_sha('easybuild-easyblocks', 'easybuilders', github_user=GITHUB_TEST_ACCOUNT, branch='develop') self.assertTrue(re.match('^[0-9a-f]{40}$', sha)) + @requires_github_token() def test_github_download_repo(self): """Test download_repo function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - cwd = os.getcwd() self.mock_stdout(True) @@ -660,11 +651,10 @@ def test_github_download_repo_commit(self): path=self.test_prefix, commit='0000000000000000000000000000000000000000') @unittest.skipIf(not HAVE_KEYRING, "keyring module not available") + @requires_github_token() def test_install_github_token(self): """Test for install_github_token function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - + random_user = ''.join(random.choice(ascii_letters) for _ in range(10)) random_user = ''.join(random.choice(ascii_letters) for _ in range(10)) self.assertEqual(gh.fetch_github_token(random_user), None) @@ -696,11 +686,9 @@ def fake_getpass(*args, **kwargs): self.assertTrue(token == self.github_token) @unittest.skipIf(not HAVE_KEYRING, "keyring module not available") + @requires_github_token() def test_validate_github_token(self): """Test for validate_github_token function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - self.assertTrue(gh.validate_github_token(self.github_token, GITHUB_TEST_ACCOUNT)) # if a token in the old format is available, test with that too @@ -713,11 +701,9 @@ def test_validate_github_token(self): if finegrained_token: self.assertTrue(gh.validate_github_token(finegrained_token, GITHUB_TEST_ACCOUNT)) + @requires_github_token() def test_github_find_easybuild_easyconfig(self): """Test for find_easybuild_easyconfig function""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - with self.mocked_stdout_stderr(): path = gh.find_easybuild_easyconfig(github_user=GITHUB_TEST_ACCOUNT) expected = os.path.join('e', 'EasyBuild', r'EasyBuild-[1-9]+\.[0-9]+\.[0-9]+\.eb') @@ -759,12 +745,9 @@ def test_github_find_patches(self): self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None) self.mock_stdout(False) + @requires_github_token() def test_github_det_commit_status(self): """Test det_commit_status function.""" - - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - # ancient commit, from Jenkins era, no commit status available anymore commit_sha = 'ec5d6f7191676a86a18404616691796a352c5f1d' res = gh.det_commit_status('easybuilders', 'easybuild-easyconfigs', commit_sha, GITHUB_TEST_ACCOUNT) @@ -1062,11 +1045,9 @@ def test_github_det_patch_specs(self): self.assertEqual([i[0] for i in res], patch_paths) self.assertEqual([i[1] for i in res], ['A'] + ['patched_bundle'] * 4) + @requires_github_token() def test_github_restclient(self): """Test use of RestClient.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - client = RestClient('https://api.github.com', username=GITHUB_TEST_ACCOUNT, token=self.github_token) status, body = client.repos['easybuilders']['testrepository'].contents.a_directory['a_file.txt'].get() @@ -1096,22 +1077,18 @@ def test_github_restclient(self): httperror_hit = True self.assertTrue(httperror_hit, "expected HTTPError not encountered") + @requires_github_token() def test_github_create_delete_gist(self): """Test create_gist and delete_gist.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - test_txt = "This is just a test." gist_url = gh.create_gist(test_txt, 'test.txt', github_user=GITHUB_TEST_ACCOUNT, github_token=self.github_token) gist_id = gist_url.split('/')[-1] gh.delete_gist(gist_id, github_user=GITHUB_TEST_ACCOUNT, github_token=self.github_token) + @requires_github_token() def test_github_det_account_repo_branch_for_pr(self): """Test det_account_branch_for_pr.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - init_config(build_options={ 'pr_target_account': 'easybuilders', 'pr_target_repo': 'easybuild-easyconfigs', @@ -1226,11 +1203,9 @@ def test_push_branch_to_github(self): regex = re.compile(pattern) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' doesn't match: %s" % (regex.pattern, stdout)) + @requires_github_token() def test_github_pr_test_report(self): """Test for post_pr_test_report function.""" - if self.skip_github_tests: - self.skipTest("No GitHub token available?") - init_config(build_options={ 'dry_run': True, 'github_user': GITHUB_TEST_ACCOUNT, diff --git a/test/framework/options.py b/test/framework/options.py index 22472b3db5..010710098c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -66,8 +66,8 @@ from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import DARWIN, HAVE_ARCHSPEC, get_os_type from easybuild.tools.version import VERSION +from test.framework.github import ignore_rate_limit_in_pr, requires_github_token from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config, requires_pycodestyle -from test.framework.github import ignore_rate_limit_in_pr EXTERNAL_MODULES_METADATA = """[foobar/1.2.3] name = foo, bar @@ -100,7 +100,6 @@ class CommandLineOptionsTest(EnhancedTestCase): def setUp(self): """Set up test.""" super().setUp() - self.github_token = fetch_github_token(GITHUB_TEST_ACCOUNT) self.orig_terminal_supports_colors = easybuild.tools.options.terminal_supports_colors self.orig_os_getuid = easybuild.main.os.getuid @@ -1397,11 +1396,9 @@ def check_copied_files(): with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) + @requires_github_token() def test_github_copy_ec_from_pr(self): """Test combination of --copy-ec with --from-pr.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - test_working_dir = os.path.join(self.test_prefix, 'test_working_dir') mkdir(test_working_dir) test_target_dir = os.path.join(self.test_prefix, 'test_target_dir') @@ -2020,11 +2017,9 @@ def test_dry_run_categorized(self): if os.path.exists(dummylogfn): os.remove(dummylogfn) + @requires_github_token() def test_github_from_pr(self): """Test fetching easyconfigs from a PR.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -2088,11 +2083,9 @@ def test_github_from_pr(self): except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) + @requires_github_token() def test_github_from_pr_token_log(self): """Check that --from-pr doesn't leak GitHub token in log.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -2105,23 +2098,22 @@ def test_github_from_pr_token_log(self): '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user ] + github_token = fetch_github_token(GITHUB_TEST_ACCOUNT) try: with self.mocked_stdout_stderr(): outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) stdout = self.get_stdout() stderr = self.get_stderr() - self.assertNotIn(self.github_token, outtxt) - self.assertNotIn(self.github_token, stdout) - self.assertNotIn(self.github_token, stderr) + self.assertNotIn(github_token, outtxt) + self.assertNotIn(github_token, stdout) + self.assertNotIn(github_token, stderr) except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) + @requires_github_token() def test_github_from_pr_listed_ecs(self): """Test --from-pr in combination with specifying easyconfigs on the command line.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -2169,11 +2161,9 @@ def test_github_from_pr_listed_ecs(self): except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) + @requires_github_token() def test_github_from_pr_x(self): """Test combination of --from-pr with --extended-dry-run.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -2204,6 +2194,7 @@ def test_github_from_pr_x(self): except URLError as err: print("Ignoring URLError '%s' in test_from_pr_x" % err) + @requires_github_token() def test_from_commit(self): """Test for --from-commit.""" # --from-commit does not involve using GitHub API, so no GitHub token required; @@ -2211,9 +2202,6 @@ def test_from_commit(self): # (see also https://github.blog/changelog/2025-05-08-updated-rate-limits-for-unauthenticated-requests/), # we only run this test when a GitHub token is available, # which is only the case for a limited number of test configurations (see .github/workflows/unit_tests.yml) - if self.github_token is None: - print("Skipping test_from_commit (no GitHub token available)") - return # easyconfigs commit to add EasyBuild-4.8.2.eb test_commit = '7c83a553950c233943c7b0189762f8c05cfea852' @@ -2293,6 +2281,7 @@ def test_from_commit(self): # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... + @requires_github_token() def test_xxx_include_easyblocks_from_commit(self): """Test for --include-easyblocks-from-commit.""" # --from-commit does not involve using GitHub API, so no GitHub token required; @@ -2300,9 +2289,6 @@ def test_xxx_include_easyblocks_from_commit(self): # (see also https://github.blog/changelog/2025-05-08-updated-rate-limits-for-unauthenticated-requests/), # we only run this test when a GitHub token is available, # which is only the case for a limited number of test configurations (see .github/workflows/unit_tests.yml) - if self.github_token is None: - print("Skipping test_xxx_include_easyblocks_from_commit (no GitHub token available)") - return orig_local_sys_path = sys.path[:] # easyblocks commit only touching Binary easyblock @@ -3855,11 +3841,9 @@ def test_xxx_include_generic_easyblocks(self): # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... + @requires_github_token() def test_github_xxx_include_easyblocks_from_pr(self): """Test --include-easyblocks-from-pr.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - orig_local_sys_path = sys.path[:] fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -4201,11 +4185,9 @@ def test_cleanup_tmpdir(self): tweaked_dir = os.path.join(tmpdir, tmpdir_files[0], 'tweaked_easyconfigs') self.assertExists(os.path.join(tweaked_dir, 'toy-1.0.eb')) + @requires_github_token() def test_github_preview_pr(self): """Test --preview-pr.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') eb_file = os.path.join(test_ecs_path, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb') args = [ @@ -4219,11 +4201,9 @@ def test_github_preview_pr(self): txt = self.get_stdout() self.assertRegex(txt, r"^Comparing bzip2-1.0.6\S* with bzip2-1.0.8") + @requires_github_token() def test_github_review_pr(self): """Test --review-pr.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - # PR for bwidget 1.10.1 easyconfig, see https://github.com/easybuilders/easybuild-easyconfigs/pull/22227 args = [ '--color=never', @@ -4470,11 +4450,9 @@ def _run_mock_eb(self, args, strip=False, **kwargs): stderr_txt = stderr_txt.strip() return stdout_txt, stderr_txt + @requires_github_token() def test_new_branch_github(self): """Test for --new-branch-github.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - topdir = os.path.dirname(os.path.abspath(__file__)) # test easyconfigs @@ -4545,11 +4523,9 @@ def test_new_branch_github(self): ] self.assert_multi_regex(regexs, txt) + @requires_github_token() def test_github_new_pr_from_branch(self): """Test --new-pr-from-branch.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - # see https://github.com/boegel/easybuild-easyconfigs/tree/test_new_pr_from_branch_DO_NOT_REMOVE # branch created specifically for this test, # only adds toy-0.0.eb test easyconfig compared to central develop branch @@ -4584,11 +4560,9 @@ def test_github_new_pr_from_branch(self): ] self.assert_multi_regex(regexs, txt) + @requires_github_token() def test_update_branch_github(self): """Test --update-branch-github.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - topdir = os.path.dirname(os.path.abspath(__file__)) test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') @@ -4613,11 +4587,9 @@ def test_update_branch_github(self): ] self.assert_multi_regex(regexs, txt) + @requires_github_token() def test_github_new_update_pr(self): """Test use of --new-pr (dry run only).""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - # copy toy test easyconfig topdir = os.path.dirname(os.path.abspath(__file__)) test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') @@ -4839,12 +4811,9 @@ def test_github_new_update_pr(self): ] self.assert_multi_regex(regexs, txt, assert_true=False) + @requires_github_token() def test_github_new_pr_warning_missing_patch(self): """Test warning printed by --new-pr (dry run only) when a specified patch file could not be found.""" - - if self.github_token is None: - self.skipTest("No GitHub token available?") - topdir = os.path.dirname(os.path.abspath(__file__)) test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -4880,11 +4849,9 @@ def test_github_new_pr_warning_missing_patch(self): self.assertRegex(stdout, new_pr_out_regex) self.assertRegex(stderr, warning_regex) + @requires_github_token() def test_github_sync_pr_with_develop(self): """Test use of --sync-pr-with-develop (dry run only).""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - # use https://github.com/easybuilders/easybuild-easyconfigs/pull/9150, # which is a PR from boegel:develop to easybuilders:develop # (to sync 'develop' branch in boegel's fork with central develop branch); @@ -4908,11 +4875,9 @@ def test_github_sync_pr_with_develop(self): ]) self.assertTrue(re.match(pattern, txt), "Pattern '%s' doesn't match: %s" % (pattern, txt)) + @requires_github_token() def test_github_sync_branch_with_develop(self): """Test use of --sync-branch-with-develop (dry run only).""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - # see https://github.com/boegel/easybuild-easyconfigs/tree/test_new_pr_from_branch_DO_NOT_REMOVE test_branch = 'test_new_pr_from_branch_DO_NOT_REMOVE' @@ -4936,11 +4901,9 @@ def test_github_sync_branch_with_develop(self): ]) self.assertTrue(re.match(pattern, stdout), "Pattern '%s' doesn't match: %s" % (pattern, stdout)) + @requires_github_token() def test_github_new_pr_python(self): """Check generated PR title for --new-pr on easyconfig that includes Python dependency.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - # copy toy test easyconfig test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') toy_ec = os.path.join(self.test_prefix, 'toy.eb') @@ -4980,12 +4943,9 @@ def test_github_new_pr_python(self): regex = re.compile(r"^\* title: \"\{tools\}\[system/system\] toy v0.0\"$", re.M) self.assertRegex(txt, regex) + @requires_github_token() def test_github_new_pr_delete(self): """Test use of --new-pr to delete easyconfigs.""" - - if self.github_token is None: - self.skipTest("No GitHub token available?") - ec_name = 'bzip2-1.0.8.eb' args = [ '--new-pr', @@ -5005,12 +4965,9 @@ def test_github_new_pr_delete(self): ] self.assert_multi_regex(regexs, txt) + @requires_github_token() def test_github_new_pr_dependencies(self): """Test use of --new-pr with automatic dependency lookup.""" - - if self.github_token is None: - self.skipTest("No GitHub token available?") - foo_eb = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "foo"', @@ -5052,14 +5009,11 @@ def test_github_new_pr_dependencies(self): self.assert_multi_regex(regexs, txt) + @requires_github_token() def test_github_new_pr_easyblock(self): """ Test using --new-pr to open an easyblocks PR """ - - if self.github_token is None: - self.skipTest("SNo GitHub token available?") - topdir = os.path.dirname(os.path.abspath(__file__)) toy_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') self.assertExists(toy_eb) @@ -5080,12 +5034,10 @@ def test_github_new_pr_easyblock(self): ] self.assert_multi_regex(patterns, txt) + @requires_github_token() def test_github_merge_pr(self): """ Test use of --merge-pr (dry run)""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - # start by making sure --merge-pr without dry-run errors out for a closed PR args = [ '--merge-pr', @@ -5186,11 +5138,9 @@ def test_github_merge_pr(self): ]) self.assertIn(expected_stdout, stdout) + @requires_github_token() def test_github_empty_pr(self): """Test use of --new-pr (dry run only) with no changes""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - # get file from develop branch full_url = URL_SEPARATOR.join([GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, 'develop/easybuild/easyconfigs/z/zlib/zlib-1.3.1-GCCcore-14.2.0.eb']) diff --git a/test/framework/robot.py b/test/framework/robot.py index fbbc43ec64..ca71faf11b 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -51,10 +51,10 @@ from easybuild.tools.config import module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import copy_file, mkdir, read_file, write_file -from easybuild.tools.github import fetch_github_token from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.modules import invalidate_module_caches_for, reset_module_caches from easybuild.tools.robot import check_conflicts, det_robot_path, resolve_dependencies, search_easyconfigs +from test.framework.github import requires_github_token from test.framework.utilities import find_full_path @@ -113,7 +113,6 @@ def install_mock_module(self): def setUp(self): """Set up test.""" super().setUp() - self.github_token = fetch_github_token(GITHUB_TEST_ACCOUNT) self.orig_experimental = easybuild.framework.easyconfig.tools._log.experimental self.orig_modtool = self.modtool @@ -753,11 +752,9 @@ def test_github_det_easyconfig_paths_from_commit(self): regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + @requires_github_token() def test_github_det_easyconfig_paths_from_pr(self): """Test det_easyconfig_paths function, with --from-pr enabled as well.""" - if self.github_token is None: - self.skipTest("No GitHub token available?") - fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) From 818928b5e4fe0ee12a024acdac0eec71cd7cefff Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 6 Feb 2023 12:15:07 +0100 Subject: [PATCH 06/10] Introduce `skip_silent*` decorators As that is used a lot use that for better describing the purpose. --- test/framework/github.py | 29 +++++++++++---------------- test/framework/utilities.py | 39 +++++++++++++++++++++++++------------ 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index c66faf303c..37e56e683a 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -37,7 +37,8 @@ import textwrap import unittest from string import ascii_letters -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework.utilities import (EnhancedTestCase, TestLoaderFiltered, init_config, + skip_never, skip_silentCI_unless) from time import gmtime from unittest import TextTestRunner from urllib.request import HTTPError, URLError @@ -74,14 +75,13 @@ def requires_github_access(): - """Silently skip for pull requests unless $FORCE_EB_GITHUB_TESTS is set + """Skip for pull requests unless $FORCE_EB_GITHUB_TESTS is set Useful when the test uses e.g. `git` commands to download from Github and would run into rate limits """ - return unittest.skipUnless( - os.environ.get('FORCE_EB_GITHUB_TESTS', '0') != '0' or os.getenv('GITHUB_EVENT_NAME') != 'pull_request', - "Skipping test requiring GitHub access" - ) + if 'FORCE_EB_GITHUB_TESTS' in os.environ: + return skip_never + return skip_silentCI_unless(os.getenv('GITHUB_EVENT_NAME') != 'pull_request', "Requires GitHub access") def ignore_rate_limit_in_pr(test_item): @@ -103,20 +103,13 @@ def skip_wrapper(self, *args, **kwargs): def requires_github_token(): """Require a github token to be available unless $FORCE_EB_GITHUB_TESTS is set""" if 'FORCE_EB_GITHUB_TESTS' in os.environ: - return unittest.skipIf(False, None) + return skip_never github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) - if os.getenv('GITHUB_EVENT_NAME') != 'pull_request': - return unittest.skipUnless(github_token, "No GitHub token available") - elif github_token: - return unittest.skipIf(False, None) - else: + if os.getenv('GITHUB_EVENT_NAME') == 'pull_request': # For pull requests silently skip if no token is available as that is expected - def decorator(test_item): - @functools.wraps(test_item) - def skip_wrapper(*args, **kwargs): - return - return skip_wrapper - return decorator + return skip_silentCI_unless(github_token, "No GitHub token available") + else: + return unittest.skipUnless(github_token, "No GitHub token available") class GithubTest(EnhancedTestCase): diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 31f3515c68..2942ee3cc9 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -558,6 +558,29 @@ def find_full_path(base_path, trim=(lambda x: x)): return full_path +def skip_silently(test_item): + """Decorator to turn a test into a no-op""" + @functools.wraps(test_item) + def skip_wrapper(*args, **kwargs): + return + return skip_wrapper + + +def skip_never(test_item): + """Decorator to not skip a test""" + return test_item + + +def skip_silentCI_unless(condition, reason): + """Decorator to skip a test if the condition is met. + + On CI the test is turned into a no-op to avoid any output.""" + if 'CI' in os.environ: + return skip_never if condition else skip_silently + else: + return unittest.skipUnless(condition, reason) + + def requires_pycodestyle(): try: import pycodestyle # noqa @@ -596,21 +619,13 @@ def requires_pygraph(): def requires_pysvn(): try: - from pysvn import ClientError # noqa + from pysvn import ClientError # noqa ok = True except ImportError: ok = False - if 'CI' in os.environ: - # For CI skip silently, not easy enough to install, - # see https://github.com/leafvmaple/pysvn/issues/1 - def decorator(test_item): - @functools.wraps(test_item) - def skip_wrapper(*args, **kwargs): - return - return skip_wrapper - return decorator - else: - return unittest.skipUnless(ok, "PySVN is not available, use e.g. apt-get install python3-svn") + # For CI skip silently, not easy enough to install, + # see https://github.com/leafvmaple/pysvn/issues/1 + return skip_silentCI_unless(ok, "PySVN is not available, use e.g. apt-get install python3-svn") def requires_PyYAML(): From 3856f606c19e3bf89737a47a02520ecdf92d664a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 6 Feb 2023 12:18:53 +0100 Subject: [PATCH 07/10] Silently skip GC3PIe test w/ Python 3.11 GC3Pie on Python 3.11 does not work: https://github.com/gc3pie/gc3pie/issues/674 So avoid the skip mark for that. --- test/framework/utilities.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 2942ee3cc9..e44a5cb12a 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -49,7 +49,7 @@ from easybuild.framework.easyconfig import easyconfig from easybuild.framework.easyblock import EasyBlock from easybuild.main import main -from easybuild.tools import config +from easybuild.tools import config, LooseVersion from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.environment import modify_env @@ -605,7 +605,12 @@ def requires_GC3Pie(): ok = True except ImportError: ok = False - return unittest.skipUnless(ok, "GC3Pie not available") + if LooseVersion(sys.version) < '3.11': + return unittest.skipUnless(ok, "GC3Pie not available") + else: + # GC3Pie not available for Python 3.11 so silently skip: + # https://github.com/gc3pie/gc3pie/issues/674 + return skip_silentCI_unless(ok, "GC3Pie not available") def requires_pygraph(): From defa7a074fe2d3f3b07187a6e8d2c47f4b912e42 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 20 Nov 2025 17:45:12 +0100 Subject: [PATCH 08/10] Show skipped tests at end of CI run with reason. On CI we don't want the full output of --verbose but fail when tests are skipped. This is only indicated by a "s" inbetween many dots. So show a list of skipped tests and their reason at the end. This also allows easy filtering. --- test/framework/suite.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/test/framework/suite.py b/test/framework/suite.py index d9a2454b4e..d5a1cc8567 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -137,12 +137,29 @@ def run(self, *args, **kwargs): return res -def load_tests(loader, tests, pattern): +class SkipSummaryResult(unittest.TextTestResult): + """Decorator that prints skipped tests at the end of the run if in CI environment.""" + def __init__(self, stream, descriptions, verbosity, *args, **kwargs): + super().__init__(stream, descriptions, verbosity, *args, **kwargs) + self._verbosity = verbosity + + def stopTestRun(self): + super().stopTestRun() + if 'CI' in os.environ and self.skipped: + print('\n=== Skipped tests ===') + print('\n'.join(f'\t{test}: {reason}' for test, reason in self.skipped)) + + +class SkipSummaryRunner(unittest.TextTestRunner): + resultclass = SkipSummaryResult + + +def load_tests(loader, _tests, _pattern): return EasyBuildFrameworkTestSuite(loader) if __name__ == '__main__': if len(sys.argv) > 1 and sys.argv[1].startswith('-'): - unittest.main() + unittest.main(testRunner=SkipSummaryRunner()) else: - unittest.TextTestRunner().run(EasyBuildFrameworkTestSuite(None)) + SkipSummaryRunner().run(EasyBuildFrameworkTestSuite(None)) From f0a16f490826e617c046c689413593a547d6070c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 21 Nov 2025 09:56:26 +0100 Subject: [PATCH 09/10] Update IGNORE_PATTERNS --- .github/workflows/unit_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 55850600a8..9574d103f1 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -253,12 +253,12 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available" + IGNORE_PATTERNS="=== Skipped tests ===" + IGNORE_PATTERNS+="Ignoring rate limit error" IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device" IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.7" IGNORE_PATTERNS+="|from cryptography.* import " IGNORE_PATTERNS+="|Blowfish" - IGNORE_PATTERNS+="|GC3Pie not available, skipping test" IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved" IGNORE_PATTERNS+="|algorithms.TripleDES" # ignore lines with only successful ('.') and skipped ('s') tests From 757eb374974232b0a85f5f89f9849637b371efab Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 21 Nov 2025 12:53:15 +0100 Subject: [PATCH 10/10] Ignore skipped tests Successful output looks like this > ...............................................s...................................................................................... > ................................................................................................................... > ---------------------------------------------------------------------- > Ran 946 tests in 714.512s > > OK (skipped=3) We only care about the part until "Ran xxx tests" not including the divider (dashes). Also we can ignore any lines with just "s" and ".". --- .github/workflows/unit_tests.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 9574d103f1..fd3423739f 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -253,17 +253,16 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="=== Skipped tests ===" - IGNORE_PATTERNS+="Ignoring rate limit error" + IGNORE_PATTERNS="^[s.]*$" # Only PASS (.) and SKIP (s) status on line IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device" IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.7" IGNORE_PATTERNS+="|from cryptography.* import " IGNORE_PATTERNS+="|Blowfish" IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved" IGNORE_PATTERNS+="|algorithms.TripleDES" - # ignore lines with only successful ('.') and skipped ('s') tests - IGNORE_PATTERNS+="|^[\.s]+$" + + TEST_OUTPUT=$(sed -n '/------------------------------/q;p' test_framework_suite.log) # Everything up to the result divider # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches) - PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) - test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) + PRINTED_MSG=$(echo "$TEST_OUTPUT" | egrep -v "${IGNORE_PATTERNS}" || true) + [[ -z $PRINTED_MSG ]] || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) done