From 666dc5cdb6a61e24206716e9f16b690ba120cc4a Mon Sep 17 00:00:00 2001 From: Brandon Fuller Date: Thu, 14 Nov 2024 10:01:59 -0500 Subject: [PATCH 1/4] Make paths in config object absolute --- lib/common.py | 3 +++ lib/patterns_targets.py | 5 +++-- test/tests/test_suite.py | 15 ++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/common.py b/lib/common.py index 08959dc1..7cf37a3c 100644 --- a/lib/common.py +++ b/lib/common.py @@ -626,6 +626,9 @@ def load_config(config, missing_references_ok=False): handler. """ + if isinstance(config, str): + config = yaml.load(open(config), Loader=yaml.FullLoader) + # Here we populate a list of reference sections. Items later on the list # will have higher priority includes = config.get('include_references', []) diff --git a/lib/patterns_targets.py b/lib/patterns_targets.py index 542d4116..97e00cba 100644 --- a/lib/patterns_targets.py +++ b/lib/patterns_targets.py @@ -9,6 +9,7 @@ from . import common from . import chipseq from . import helpers +from pathlib import Path HERE = os.path.abspath(os.path.dirname(__file__)) @@ -46,7 +47,7 @@ def __init__(self, config, patterns, workdir=None): as relative to `workdir` """ self.path = None - self.workdir = '.' + self.workdir = Path('.').resolve() if workdir is not None: config = os.path.join(workdir, config) patterns = os.path.join(workdir, patterns) @@ -56,7 +57,7 @@ def __init__(self, config, patterns, workdir=None): self.path = config self.config = common.load_config( - common.resolve_config(config, workdir)) + common.resolve_config(config, self.workdir)) stranded = self.config.get('stranded', None) self.stranded = None diff --git a/test/tests/test_suite.py b/test/tests/test_suite.py index b1389f03..4fc93f71 100644 --- a/test/tests/test_suite.py +++ b/test/tests/test_suite.py @@ -1,7 +1,8 @@ import sys +import os import subprocess -top_level_dir = subprocess.run(["dirname $(dirname $(pwd))"], shell=True, capture_output=True, text=True).stdout.strip() -print("top level dir: ", top_level_dir) +from pathlib import Path +top_level_dir = str(Path("../../").resolve()) sys.path.insert(0, top_level_dir) import pytest from textwrap import dedent @@ -11,15 +12,15 @@ @pytest.fixture def config(request): config_path = request.param - config = common.load_config(config_path, test=True) + config = common.load_config(config_path) + os.chdir(os.path.join(helpers.get_top_level_dir(), "workflows","rnaseq")) return patterns_targets.RNASeqConfig(config, config.get('patterns', '../workflows/rnaseq/config/rnaseq_patterns.yaml')) # Call helpers.detect_layout(), which implicitly tests common.is_paired_end() -# TODO: Make assertion condition NOT hard coded in to work with current example table -@pytest.mark.parametrize("config", ['../../workflows/rnaseq/config/config.yaml'], indirect=True) -def test_is_paired_end(config): +@pytest.mark.parametrize("config, is_paired_ref", [("../../workflows/rnaseq/config/config.yaml", False)], indirect=["config"]) +def test_is_paired_end(config, is_paired_ref): is_paired = helpers.detect_layout(config.sampletable) == 'PE' - assert not is_paired, f"Test failed, is_paired = {is_paired}" + assert is_paired==is_paired_ref, f"Test failed, is_paired is {is_paired} when it should be {is_paired_ref}" def test_config_loading(tmpdir): f0 = tmpdir.mkdir('subdir').join('file0.yaml') From e034a8efae05a122c9dab9abb08816cd63f4f094 Mon Sep 17 00:00:00 2001 From: Brandon Fuller Date: Thu, 14 Nov 2024 11:42:55 -0500 Subject: [PATCH 2/4] Work on config paths and working directory --- lib/common.py | 8 ++++---- lib/patterns_targets.py | 3 ++- test/tests/test_suite.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/common.py b/lib/common.py index 7cf37a3c..1057b318 100644 --- a/lib/common.py +++ b/lib/common.py @@ -66,13 +66,13 @@ def resolve_config(config, workdir=None): if isinstance(config, str): config = yaml.load(open(config), Loader=yaml.FullLoader) - def rel(pth): - if workdir is None or os.path.isabs(pth): + def abs_path(pth): + if os.path.isabs(pth): return pth return os.path.join(workdir, pth) for key in PATH_KEYS: - if key in config: - config[key] = rel(config[key]) + if key in config and workdir: + config[key] = abs_path(config[key]) return config diff --git a/lib/patterns_targets.py b/lib/patterns_targets.py index 97e00cba..94b69251 100644 --- a/lib/patterns_targets.py +++ b/lib/patterns_targets.py @@ -48,12 +48,13 @@ def __init__(self, config, patterns, workdir=None): """ self.path = None self.workdir = Path('.').resolve() + self.workdir = None if workdir is not None: - config = os.path.join(workdir, config) patterns = os.path.join(workdir, patterns) self.workdir = workdir if isinstance(config, str): + config = os.path.join(workdir, config) self.path = config self.config = common.load_config( diff --git a/test/tests/test_suite.py b/test/tests/test_suite.py index 4fc93f71..5616fa29 100644 --- a/test/tests/test_suite.py +++ b/test/tests/test_suite.py @@ -13,8 +13,8 @@ def config(request): config_path = request.param config = common.load_config(config_path) - os.chdir(os.path.join(helpers.get_top_level_dir(), "workflows","rnaseq")) - return patterns_targets.RNASeqConfig(config, config.get('patterns', '../workflows/rnaseq/config/rnaseq_patterns.yaml')) + rnaseq_workdir = os.path.join(helpers.get_top_level_dir(), "workflows","rnaseq") + return patterns_targets.RNASeqConfig(config, config.get('patterns', '../workflows/rnaseq/config/rnaseq_patterns.yaml'), workdir=rnaseq_workdir) # Call helpers.detect_layout(), which implicitly tests common.is_paired_end() @pytest.mark.parametrize("config, is_paired_ref", [("../../workflows/rnaseq/config/config.yaml", False)], indirect=["config"]) From daf567eb78dbc8d220c04552b4fdc10797f96506 Mon Sep 17 00:00:00 2001 From: Brandon Fuller Date: Thu, 21 Nov 2024 14:21:45 -0500 Subject: [PATCH 3/4] Hook test_suite.py into lcdb-wf-test Fix code so that running lcdb-wf-test unit_tests --pytest runs the pytests contained in test_suite.py. Minor bug fixes. --- lib/common.py | 8 +++++++- lib/helpers.py | 3 +-- lib/patterns_targets.py | 1 - test/lcdb-wf-test | 1 + test/tests/test_suite.py | 7 ++++--- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/common.py b/lib/common.py index 1057b318..99abd398 100644 --- a/lib/common.py +++ b/lib/common.py @@ -27,6 +27,7 @@ 'merged_dir', 'peaks_dir', 'hub_config', + 'include_references', ] @@ -72,7 +73,11 @@ def abs_path(pth): return os.path.join(workdir, pth) for key in PATH_KEYS: if key in config and workdir: - config[key] = abs_path(config[key]) + if isinstance(config[key], list): + for i in range(len(config[key])): + config[key][i] = abs_path(config[key][i]) + else: + config[key] = abs_path(config[key]) return config @@ -629,6 +634,7 @@ def load_config(config, missing_references_ok=False): if isinstance(config, str): config = yaml.load(open(config), Loader=yaml.FullLoader) + config = resolve_config(config) # Here we populate a list of reference sections. Items later on the list # will have higher priority includes = config.get('include_references', []) diff --git a/lib/helpers.py b/lib/helpers.py index 651a9c13..9a2b914e 100644 --- a/lib/helpers.py +++ b/lib/helpers.py @@ -219,7 +219,6 @@ def get_top_level_dir(start_dir=None): if current_dir == parent_dir: break current_dir = parent_dir - #TODO: Check for other edge cases? - return None + return current_dir diff --git a/lib/patterns_targets.py b/lib/patterns_targets.py index 94b69251..9a520b6b 100644 --- a/lib/patterns_targets.py +++ b/lib/patterns_targets.py @@ -47,7 +47,6 @@ def __init__(self, config, patterns, workdir=None): as relative to `workdir` """ self.path = None - self.workdir = Path('.').resolve() self.workdir = None if workdir is not None: patterns = os.path.join(workdir, patterns) diff --git a/test/lcdb-wf-test b/test/lcdb-wf-test index df59b24c..b6af4914 100755 --- a/test/lcdb-wf-test +++ b/test/lcdb-wf-test @@ -324,6 +324,7 @@ class Runner(object): if args.pytest: print_header("pytest") sp.run(["pytest", "--doctest-modules", "lib"], check=True, cwd=TOPLEVEL) + sp.run(["pytest", "test/tests"], check=True, cwd=TOPLEVEL) if args.url_check: print_header("url check") diff --git a/test/tests/test_suite.py b/test/tests/test_suite.py index 5616fa29..f8d1665e 100644 --- a/test/tests/test_suite.py +++ b/test/tests/test_suite.py @@ -2,22 +2,23 @@ import os import subprocess from pathlib import Path -top_level_dir = str(Path("../../").resolve()) +top_level_dir = str(Path(".").resolve()) sys.path.insert(0, top_level_dir) import pytest from textwrap import dedent from lib import common, helpers, patterns_targets +os.chdir(helpers.get_top_level_dir()) # Make config object that can be re-used for any test @pytest.fixture def config(request): config_path = request.param - config = common.load_config(config_path) rnaseq_workdir = os.path.join(helpers.get_top_level_dir(), "workflows","rnaseq") + config = common.load_config(common.resolve_config(config_path, rnaseq_workdir)) return patterns_targets.RNASeqConfig(config, config.get('patterns', '../workflows/rnaseq/config/rnaseq_patterns.yaml'), workdir=rnaseq_workdir) # Call helpers.detect_layout(), which implicitly tests common.is_paired_end() -@pytest.mark.parametrize("config, is_paired_ref", [("../../workflows/rnaseq/config/config.yaml", False)], indirect=["config"]) +@pytest.mark.parametrize("config, is_paired_ref", [("workflows/rnaseq/config/config.yaml", False)], indirect=["config"]) def test_is_paired_end(config, is_paired_ref): is_paired = helpers.detect_layout(config.sampletable) == 'PE' assert is_paired==is_paired_ref, f"Test failed, is_paired is {is_paired} when it should be {is_paired_ref}" From b25ca774207d54f0f92edbc293a14f8fd511f14a Mon Sep 17 00:00:00 2001 From: Brandon Fuller Date: Thu, 21 Nov 2024 14:29:18 -0500 Subject: [PATCH 4/4] Bug fix --- test/tests/test_suite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/tests/test_suite.py b/test/tests/test_suite.py index f8d1665e..3b7e9765 100644 --- a/test/tests/test_suite.py +++ b/test/tests/test_suite.py @@ -7,7 +7,6 @@ import pytest from textwrap import dedent from lib import common, helpers, patterns_targets -os.chdir(helpers.get_top_level_dir()) # Make config object that can be re-used for any test @pytest.fixture