From ac204f86583d6c62e80996fdd67879a5c1a23465 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 30 Jul 2021 10:47:00 +0200 Subject: [PATCH 1/3] Ignore cmdline args passed to the test suite --- test/framework/utilities.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index d55540260a..2e7c2bcbd1 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -480,6 +480,8 @@ def init_config(args=None, build_options=None, with_include=True, clear_caches=T cleanup(clear_caches=clear_caches) + if args is None: + args = [] # Ignore cmdline args as those are meant for the unittest framework # initialize configuration so config.get_modules_tool function works eb_go = eboptions.parse_options(args=args, with_include=with_include) config.init(eb_go.options, eb_go.get_options_by_section('config')) From fa18381812eed81a50ef6059a3fc904b4991da07 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 30 Jul 2021 11:13:17 +0200 Subject: [PATCH 2/3] Allow passing arguments to unittest --- test/framework/asyncprocess.py | 7 +++-- test/framework/build_log.py | 7 +++-- test/framework/config.py | 7 +++-- test/framework/containers.py | 7 +++-- test/framework/docs.py | 7 +++-- test/framework/easyblock.py | 7 +++-- test/framework/easyconfig.py | 7 +++-- test/framework/easyconfigformat.py | 7 +++-- test/framework/easyconfigparser.py | 7 +++-- test/framework/easyconfigversion.py | 7 +++-- test/framework/easystack.py | 7 +++-- test/framework/ebconfigobj.py | 7 +++-- test/framework/environment.py | 7 +++-- test/framework/filetools.py | 7 +++-- test/framework/format_convert.py | 7 +++-- test/framework/general.py | 7 +++-- test/framework/github.py | 7 +++-- test/framework/hooks.py | 7 +++-- test/framework/include.py | 7 +++-- test/framework/lib.py | 7 +++-- test/framework/license.py | 7 +++-- test/framework/module_generator.py | 11 +++++-- test/framework/modules.py | 7 +++-- test/framework/modulestool.py | 7 +++-- test/framework/options.py | 7 +++-- test/framework/output.py | 7 +++-- test/framework/package.py | 7 +++-- test/framework/parallelbuild.py | 7 +++-- test/framework/repository.py | 7 +++-- test/framework/robot.py | 7 +++-- test/framework/run.py | 7 +++-- test/framework/style.py | 7 +++-- test/framework/suite.py | 43 +++++++++++++++++----------- test/framework/systemtools.py | 7 +++-- test/framework/toolchain.py | 7 +++-- test/framework/toolchainvariables.py | 7 +++-- test/framework/toy_build.py | 7 +++-- test/framework/tweak.py | 7 +++-- test/framework/type_checking.py | 7 +++-- test/framework/utilities.py | 3 +- test/framework/utilities_test.py | 7 +++-- test/framework/variables.py | 7 +++-- 42 files changed, 231 insertions(+), 99 deletions(-) diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py index d089895f1c..b5d5afe1bc 100644 --- a/test/framework/asyncprocess.py +++ b/test/framework/asyncprocess.py @@ -67,9 +67,12 @@ def tearDown(self): super().tearDown() -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(AsyncProcessTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(AsyncProcessTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(AsyncProcessTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/build_log.py b/test/framework/build_log.py index af939501ea..fc7aa83529 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -438,9 +438,12 @@ def test_raise_nosupport(self): raise_nosupport, 'foobar', 42) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(BuildLogTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(BuildLogTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(BuildLogTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/config.py b/test/framework/config.py index fbd8bf7d2c..23ccadb0bc 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -730,8 +730,11 @@ def test_update_build_options(self): self.assertEqual(build_option('pr_target_account'), 'easybuilders') -def suite(): - return TestLoaderFiltered().loadTestsFromTestCase(EasyBuildConfigTest, sys.argv[1:]) +def suite(loader=None): + if loader: + return loader.loadTestsFromTestCase(EasyBuildConfigTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(EasyBuildConfigTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/containers.py b/test/framework/containers.py index 321df783d2..b0fa41b51e 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -509,9 +509,12 @@ def test_container_config_template_recipe(self): self.assertEqual(cont_recipe, expected) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(ContainersTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(ContainersTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(ContainersTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/docs.py b/test/framework/docs.py index 4b281df14a..114155ae92 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -1407,9 +1407,12 @@ def get_eb_help_output(arg=''): self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) -def suite(): +def suite(loader=None): """ returns all test cases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(DocsTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(DocsTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(DocsTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 5d4c3d19ee..38755cb501 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -3593,9 +3593,12 @@ def custom_step(self): self.assertIn('Ran custom', logtxt) -def suite(): +def suite(loader=None): """ return all the tests in this file """ - return TestLoaderFiltered().loadTestsFromTestCase(EasyBlockTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(EasyBlockTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(EasyBlockTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 803868a51d..e68c2b74a1 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -5257,9 +5257,12 @@ def test_templates(self): self.assertRegex(stderr.getvalue(), regex) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(EasyConfigTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(EasyConfigTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(EasyConfigTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/easyconfigformat.py b/test/framework/easyconfigformat.py index 537b4c1977..d2575d9690 100644 --- a/test/framework/easyconfigformat.py +++ b/test/framework/easyconfigformat.py @@ -46,9 +46,12 @@ def test_parser_version_regex(self): self.assertEqual(version['minor'], int(res['minor'])) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(EasyConfigFormatTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(EasyConfigFormatTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(EasyConfigFormatTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index 4fbe6676a0..d04361b78f 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -222,9 +222,12 @@ def test_check_value_types(self): self.assertEqual(ecdict['version'], '1.4') -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(EasyConfigParserTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(EasyConfigParserTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(EasyConfigParserTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/easyconfigversion.py b/test/framework/easyconfigversion.py index 300cf6aff2..59cf051a42 100644 --- a/test/framework/easyconfigversion.py +++ b/test/framework/easyconfigversion.py @@ -285,9 +285,12 @@ def test_hashing(self): self.assertTrue(hash(test_case)) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(EasyConfigVersion, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(EasyConfigVersion) + else: + return TestLoaderFiltered().loadTestsFromTestCase(EasyConfigVersion, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 6b5fedf68a..46a9653296 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -204,9 +204,12 @@ def test_check_value(self): self.assertErrorRegex(EasyBuildError, error_pattern, check_value, version, context) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(EasyStackTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(EasyStackTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(EasyStackTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/ebconfigobj.py b/test/framework/ebconfigobj.py index 048b356193..113c9e557d 100644 --- a/test/framework/ebconfigobj.py +++ b/test/framework/ebconfigobj.py @@ -279,9 +279,12 @@ def test_ebconfigobj(self): self.assertEqual(res, {'foo': 'bar'}) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(TestEBConfigObj, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(TestEBConfigObj) + else: + return TestLoaderFiltered().loadTestsFromTestCase(TestEBConfigObj, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/environment.py b/test/framework/environment.py index 75919f6a13..6c1c05ee8e 100644 --- a/test/framework/environment.py +++ b/test/framework/environment.py @@ -162,9 +162,12 @@ def test_sanitize_env(self): self.assertEqual(os.getenv('LD_PRELOAD'), None) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(EnvironmentTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(EnvironmentTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(EnvironmentTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ca2b9bcb47..ea0b877b4d 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -4159,9 +4159,12 @@ def expected_suffix(n_calls_to_create_non_existing_paths): ft.remove_dir(test_root) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(FileToolsTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(FileToolsTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(FileToolsTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py index fc028f7299..3822c7fcf1 100644 --- a/test/framework/format_convert.py +++ b/test/framework/format_convert.py @@ -60,9 +60,12 @@ def test_dependency(self): self.assertEqual(str(res), txt) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(ConvertTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(ConvertTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(ConvertTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/general.py b/test/framework/general.py index e3b15129cc..fe08c3a960 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -155,9 +155,12 @@ def test_import_available_modules(self): self.assertEqual([test123.one, test123.three, test123.two], res) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(GeneralTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(GeneralTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(GeneralTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/github.py b/test/framework/github.py index 20fff8c2fd..3668e4d187 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -1459,9 +1459,12 @@ def test_is_patch_for(self): self.assertTrue(is_patch_for('ext_foo-1.2.3.patch', ec)) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(GithubTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(GithubTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(GithubTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/hooks.py b/test/framework/hooks.py index fbd001ff5c..b460842e33 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -324,9 +324,12 @@ def install_hook(self): self.assertErrorRegex(EasyBuildError, error_msg_pattern, load_hooks, test_broken_hooks_pymod) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(HooksTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(HooksTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(HooksTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/include.py b/test/framework/include.py index ef20e682fd..c65c1e17ec 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -258,9 +258,12 @@ def test_is_software_specific_easyblock(self): self.assertFalse(is_software_specific_easyblock(os.path.join(test_easyblocks, 'generic', 'toolchain.py'))) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(IncludeTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(IncludeTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(IncludeTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/lib.py b/test/framework/lib.py index b9771a41e2..88b8264cec 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -135,8 +135,11 @@ def test_modules_tool(self): self.assertEqual(modtool.list(), [{'default': None, 'mod_name': 'GCC/6.4.0-2.28'}]) -def suite(): - return TestLoaderFiltered().loadTestsFromTestCase(EasyBuildLibTest, sys.argv[1:]) +def suite(loader=None): + if loader: + return loader.loadTestsFromTestCase(EasyBuildLibTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(EasyBuildLibTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/license.py b/test/framework/license.py index 66e5938ab7..b189e3111d 100644 --- a/test/framework/license.py +++ b/test/framework/license.py @@ -66,9 +66,12 @@ def test_licenses(self): self.assertTrue(issubclass(lics[lic], License)) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(LicenseTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(LicenseTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(LicenseTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 0196860d40..eac101ca7b 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1757,11 +1757,16 @@ class LuaModuleGeneratorTest(ModuleGeneratorTest): MODULE_GENERATOR_CLASS = ModuleGeneratorLua -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ + if loader: + args = [] + else: + loader = TestLoaderFiltered() + args = [sys.argv[1:]] suite = TestSuite() - suite.addTests(TestLoaderFiltered().loadTestsFromTestCase(TclModuleGeneratorTest, sys.argv[1:])) - suite.addTests(TestLoaderFiltered().loadTestsFromTestCase(LuaModuleGeneratorTest, sys.argv[1:])) + suite.addTests(loader.loadTestsFromTestCase(TclModuleGeneratorTest, *args)) + suite.addTests(loader.loadTestsFromTestCase(LuaModuleGeneratorTest, *args)) return suite diff --git a/test/framework/modules.py b/test/framework/modules.py index dc111930ac..b5d345fb9c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -2056,9 +2056,12 @@ def test_module_load_environment(self): self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases=['some', 'list']) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(ModulesTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(ModulesTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(ModulesTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 518714658b..ad89b4e3e5 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -259,9 +259,12 @@ def tearDown(self): os.environ.pop('module', None) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(ModulesToolTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(ModulesToolTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(ModulesToolTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/options.py b/test/framework/options.py index 6ae9383546..744b441dca 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -7463,9 +7463,12 @@ def test_opts_dict_to_eb_opts(self): self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(CommandLineOptionsTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(CommandLineOptionsTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(CommandLineOptionsTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/output.py b/test/framework/output.py index 71d5715c69..c4ff0740fb 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -204,9 +204,12 @@ def test_get_start_update_stop_progress_bar(self): stop_progress_bar(PROGRESS_BAR_EXTENSIONS) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(OutputTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(OutputTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(OutputTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/package.py b/test/framework/package.py index d230ae265e..c18bfc8fc5 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -269,9 +269,12 @@ def test_package(self): self.mock_stdout(False) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(PackageTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(PackageTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(PackageTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index e89dd2a8e3..0c821dd882 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -389,9 +389,12 @@ def test_build_easyconfigs_in_parallel_slurm(self): self.assertEqual(jobs[1].job_specs, expected) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(ParallelBuildTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(ParallelBuildTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(ParallelBuildTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/repository.py b/test/framework/repository.py index 488d6c1e91..e6c76dd564 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -170,9 +170,12 @@ def tearDown(self): shutil.rmtree(self.path, True) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(RepositoryTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(RepositoryTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(RepositoryTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/robot.py b/test/framework/robot.py index 5c9843526c..895c92f834 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1620,9 +1620,12 @@ def test_search_easyconfigs(self): self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(RobotTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(RobotTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(RobotTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/run.py b/test/framework/run.py index 74f494527e..a06e849a74 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -2253,9 +2253,12 @@ def test_run_cmd_sysroot(self): self.assertEqual(out, "hello\n") -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(RunTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(RunTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(RunTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/style.py b/test/framework/style.py index 0c1d9b140a..6305052619 100644 --- a/test/framework/style.py +++ b/test/framework/style.py @@ -106,9 +106,12 @@ def test_check_trailing_whitespace(self): self.assertEqual(result, expected_result) -def suite(): +def suite(loader=None): """Return all style tests for easyconfigs.""" - return TestLoaderFiltered().loadTestsFromTestCase(StyleTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(StyleTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(StyleTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/suite.py b/test/framework/suite.py index afec127c83..63da252cc2 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -110,27 +110,36 @@ sys.stderr.write("No execution rights on temporary files, specify another location via $TMPDIR: %s\n" % err) sys.exit(1) -# initialize logger for all the unit tests -fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log') -os.close(fd) -os.remove(log_fn) -fancylogger.logToFile(log_fn) -log = fancylogger.getLogger() - # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, d, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, tw, p, i, pkg, env, et, st, h, ct, lib, u, es, ou] -SUITE = unittest.TestSuite([x.suite() for x in tests]) -res = unittest.TextTestRunner().run(SUITE) -fancylogger.logToFile(log_fn, enable=False) +class EasyBuildFrameworkTestSuite(unittest.TestSuite): + def __init__(self, loader): + super(EasyBuildFrameworkTestSuite, self).__init__([x.suite(loader) for x in tests]) + + def run(self, *args, **kwargs): + # initialize logger for all the unit tests + fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log') + os.close(fd) + os.remove(log_fn) + fancylogger.logToFile(log_fn) + res = super(EasyBuildFrameworkTestSuite, self).run(*args, **kwargs) + fancylogger.logToFile(log_fn, enable=False) + if not res.wasSuccessful(): + sys.stderr.write("ERROR: Not all tests were successful.\n") + print("Log available at %s" % log_fn) + else: + for fn in glob.glob('%s*' % log_fn): + os.remove(fn) + return res + + +def load_tests(loader, tests, pattern): + return EasyBuildFrameworkTestSuite(loader) + -if not res.wasSuccessful(): - sys.stderr.write("ERROR: Not all tests were successful.\n") - print("Log available at %s" % log_fn) - sys.exit(2) -else: - for fn in glob.glob('%s*' % log_fn): - os.remove(fn) +if __name__ == '__main__': + unittest.main() diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 8c49a82a71..b73443947c 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1454,9 +1454,12 @@ def test_get_cuda_architectures(self): modify_env(os.environ, start_env, verbose=False) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(SystemToolsTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(SystemToolsTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(SystemToolsTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 02fbf044f5..6e6910a640 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -3280,9 +3280,12 @@ def test_get_flag(self): self.assertEqual(tc.get_flag('openmp'), flagstring) -def suite(): +def suite(loader=None): """ return all the tests""" - return TestLoaderFiltered().loadTestsFromTestCase(ToolchainTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(ToolchainTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(ToolchainTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/toolchainvariables.py b/test/framework/toolchainvariables.py index 4ad56d904e..4528018314 100644 --- a/test/framework/toolchainvariables.py +++ b/test/framework/toolchainvariables.py @@ -163,9 +163,12 @@ class TCV(ToolchainVariables): self.assertEqual(str(tcv['MPICH_CC']), "icc -test") -def suite(): +def suite(loader=None): """ return all the tests""" - return TestLoaderFiltered().loadTestsFromTestCase(ToolchainVariablesTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(ToolchainVariablesTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(ToolchainVariablesTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 914daa306e..3e6d124d06 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -5000,9 +5000,12 @@ def test_toy_multiple_ecs_module(self): f"Pattern '{regex.pattern}' should *not* be found in: {toy_app_modtxt}") -def suite(): +def suite(loader=None): """ return all the tests in this file """ - return TestLoaderFiltered().loadTestsFromTestCase(ToyBuildTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(ToyBuildTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(ToyBuildTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/tweak.py b/test/framework/tweak.py index dcc3d8dd5e..71d73dde8a 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -541,9 +541,12 @@ def test_list_deps_versionsuffixes(self): self.assertEqual(list_deps_versionsuffixes(ec_spec), ['-deps']) -def suite(): +def suite(loader=None): """ return all the tests in this file """ - return TestLoaderFiltered().loadTestsFromTestCase(TweakTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(TweakTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(TweakTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index a9ce5df15b..7fe44e1ee8 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -840,9 +840,12 @@ def test_ensure_iterable_license_specs(self): self.assertErrorRegex(EasyBuildError, error_msg, ensure_iterable_license_specs, (42, 'foo')) -def suite(): +def suite(loader=None): """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(TypeCheckingTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(TypeCheckingTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(TypeCheckingTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 2e7c2bcbd1..eb93170c87 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -66,8 +66,9 @@ if key.startswith('%s_' % CONFIG_ENV_VAR_PREFIX): del os.environ[key] +# Ignore cmdline args as those are meant for the unittest framework # ignore any existing configuration files -go = EasyBuildOptions(go_useconfigfiles=False) +go = EasyBuildOptions(go_args=[], go_useconfigfiles=False) os.environ['EASYBUILD_IGNORECONFIGFILES'] = ','.join(go.options.configfiles) # redefine $TEST_EASYBUILD_X env vars as $EASYBUILD_X diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 64af27ce91..fe825001a2 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -224,9 +224,12 @@ def test_unique_ordered_extend(self): self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, reference) -def suite(): +def suite(loader=None): """ return all the tests in this file """ - return TestLoaderFiltered().loadTestsFromTestCase(UtilitiesTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(UtilitiesTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(UtilitiesTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/variables.py b/test/framework/variables.py index 0ff7428e94..4ca582e626 100644 --- a/test/framework/variables.py +++ b/test/framework/variables.py @@ -90,9 +90,12 @@ def test_empty_variables(self): self.assertEqual(v['FOOBAR'], []) -def suite(): +def suite(loader=None): """ return all the tests""" - return TestLoaderFiltered().loadTestsFromTestCase(VariablesTest, sys.argv[1:]) + if loader: + return loader.loadTestsFromTestCase(VariablesTest) + else: + return TestLoaderFiltered().loadTestsFromTestCase(VariablesTest, sys.argv[1:]) if __name__ == '__main__': From 9623fe6dcf39b252b9b38e5805ed74374fe02f20 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 4 Mar 2025 17:48:42 +0100 Subject: [PATCH 3/3] Support oldstyle filtering --- test/framework/suite.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/framework/suite.py b/test/framework/suite.py index 63da252cc2..d9a2454b4e 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -118,7 +118,7 @@ class EasyBuildFrameworkTestSuite(unittest.TestSuite): def __init__(self, loader): - super(EasyBuildFrameworkTestSuite, self).__init__([x.suite(loader) for x in tests]) + super().__init__([x.suite(loader) for x in tests]) def run(self, *args, **kwargs): # initialize logger for all the unit tests @@ -126,7 +126,7 @@ def run(self, *args, **kwargs): os.close(fd) os.remove(log_fn) fancylogger.logToFile(log_fn) - res = super(EasyBuildFrameworkTestSuite, self).run(*args, **kwargs) + res = super().run(*args, **kwargs) fancylogger.logToFile(log_fn, enable=False) if not res.wasSuccessful(): sys.stderr.write("ERROR: Not all tests were successful.\n") @@ -142,4 +142,7 @@ def load_tests(loader, tests, pattern): if __name__ == '__main__': - unittest.main() + if len(sys.argv) > 1 and sys.argv[1].startswith('-'): + unittest.main() + else: + unittest.TextTestRunner().run(EasyBuildFrameworkTestSuite(None))