[DRAFT] refactor

.gitignore

@@ -66,3 +66,6 @@ workflows/rnaseq/downstream/rnaseq.html
 ._*
 Rplots.pdf
 /lib/include/*
+
+workflows/*/references
+

ci/get-data.py

@@ -1,37 +1,112 @@
 #!/usr/bin/env python
+import argparse
 import os
+
 from snakemake.shell import shell
 from snakemake.utils import makedirs
 
-shell.executable('/bin/bash')
-BRANCH = 'master'
-URL = 'https://github.com/lcdb/lcdb-test-data/blob/{0}/data/{{}}?raw=true'.format(BRANCH)
+BRANCH = "master"
+URL = "https://github.com/lcdb/lcdb-test-data/blob/{0}/data/{{}}?raw=true".format(
+    BRANCH
+)
+
+TOPLEVEL = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 
-def _download_file(fn, dest=None):
+def _download_file(fn, dest=None, verbose=False):
     url = URL.format(fn)
     if dest is None:
         dest = fn
+    dest = os.path.join(TOPLEVEL, dest)
     makedirs(os.path.dirname(dest))
-    basename = os.path.basename(fn)
-    shell('wget -q -O- {url} > {dest}')
+    if not verbose:
+        q = "-q"
+    else:
+        q = ""
+    shell(f"wget {q} -O- {url} > {dest}")
+    if verbose:
+        print(f"Saved {dest}")
     return dest
 
 
-_download_file('rnaseq_samples/sample1/sample1.small_R1.fastq.gz', 'workflows/rnaseq/data/example_data/rnaseq_sample1.fq.gz')
-_download_file('rnaseq_samples/sample2/sample2.small_R1.fastq.gz', 'workflows/rnaseq/data/example_data/rnaseq_sample2.fq.gz')
-_download_file('rnaseq_samples/sample3/sample3.small_R1.fastq.gz', 'workflows/rnaseq/data/example_data/rnaseq_sample3.fq.gz')
-_download_file('rnaseq_samples/sample4/sample4.small_R1.fastq.gz', 'workflows/rnaseq/data/example_data/rnaseq_sample4.fq.gz')
+ap = argparse.ArgumentParser()
+ap.add_argument("-v", "--verbose", action="store_true", help="Be verbose when downloading")
+args = ap.parse_args()
+
+_download_file(
+    "rnaseq_samples/sample1/sample1.small_R1.fastq.gz",
+    "workflows/rnaseq/data/example_data/rnaseq_sample1.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "rnaseq_samples/sample2/sample2.small_R1.fastq.gz",
+    "workflows/rnaseq/data/example_data/rnaseq_sample2.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "rnaseq_samples/sample3/sample3.small_R1.fastq.gz",
+    "workflows/rnaseq/data/example_data/rnaseq_sample3.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "rnaseq_samples/sample4/sample4.small_R1.fastq.gz",
+    "workflows/rnaseq/data/example_data/rnaseq_sample4.fq.gz",
+    args.verbose,
+)
 
-_download_file('rnaseq_samples/sample1/sample1.small_R1.fastq.gz', 'workflows/rnaseq/data/example_data/rnaseq_sample1PE_1.fq.gz')
-_download_file('rnaseq_samples/sample1/sample1.small_R2.fastq.gz', 'workflows/rnaseq/data/example_data/rnaseq_sample1PE_2.fq.gz')
-_download_file('rnaseq_samples/sample2/sample2.small_R1.fastq.gz', 'workflows/rnaseq/data/example_data/rnaseq_sample2PE_1.fq.gz')
-_download_file('rnaseq_samples/sample2/sample2.small_R2.fastq.gz', 'workflows/rnaseq/data/example_data/rnaseq_sample2PE_2.fq.gz')
+_download_file(
+    "rnaseq_samples/sample1/sample1.small_R1.fastq.gz",
+    "workflows/rnaseq/data/example_data/rnaseq_sample1PE_1.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "rnaseq_samples/sample1/sample1.small_R2.fastq.gz",
+    "workflows/rnaseq/data/example_data/rnaseq_sample1PE_2.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "rnaseq_samples/sample2/sample2.small_R1.fastq.gz",
+    "workflows/rnaseq/data/example_data/rnaseq_sample2PE_1.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "rnaseq_samples/sample2/sample2.small_R2.fastq.gz",
+    "workflows/rnaseq/data/example_data/rnaseq_sample2PE_2.fq.gz",
+    args.verbose,
+)
 
-_download_file('chipseq_samples/input_1/input_1.tiny_R1.fastq.gz', 'workflows/chipseq/data/example_data/chipseq_input1.fq.gz')
-_download_file('chipseq_samples/ip_1/ip_1.tiny_R1.fastq.gz', 'workflows/chipseq/data/example_data/chipseq_ip1.fq.gz')
-_download_file('chipseq_samples/input_2/input_2.tiny_R1.fastq.gz', 'workflows/chipseq/data/example_data/chipseq_input2.fq.gz')
-_download_file('chipseq_samples/ip_2/ip_2.tiny_R1.fastq.gz', 'workflows/chipseq/data/example_data/chipseq_ip2.fq.gz')
-_download_file('chipseq_samples/ip_3/ip_3.tiny_R1.fastq.gz', 'workflows/chipseq/data/example_data/chipseq_ip3.fq.gz')
-_download_file('chipseq_samples/ip_4/ip_4.tiny_R1.fastq.gz', 'workflows/chipseq/data/example_data/chipseq_ip4.fq.gz')
-_download_file('chipseq_samples/input_3/input_3.tiny_R1.fastq.gz', 'workflows/chipseq/data/example_data/chipseq_input3.fq.gz')
+_download_file(
+    "chipseq_samples/input_1/input_1.tiny_R1.fastq.gz",
+    "workflows/chipseq/data/example_data/chipseq_input1.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "chipseq_samples/ip_1/ip_1.tiny_R1.fastq.gz",
+    "workflows/chipseq/data/example_data/chipseq_ip1.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "chipseq_samples/input_2/input_2.tiny_R1.fastq.gz",
+    "workflows/chipseq/data/example_data/chipseq_input2.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "chipseq_samples/ip_2/ip_2.tiny_R1.fastq.gz",
+    "workflows/chipseq/data/example_data/chipseq_ip2.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "chipseq_samples/ip_3/ip_3.tiny_R1.fastq.gz",
+    "workflows/chipseq/data/example_data/chipseq_ip3.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "chipseq_samples/ip_4/ip_4.tiny_R1.fastq.gz",
+    "workflows/chipseq/data/example_data/chipseq_ip4.fq.gz",
+    args.verbose,
+)
+_download_file(
+    "chipseq_samples/input_3/input_3.tiny_R1.fastq.gz",
+    "workflows/chipseq/data/example_data/chipseq_input3.fq.gz",
+    args.verbose,
+)

ci/preprocessor.py

@@ -7,54 +7,16 @@
 in production. Rather than require users edit files to remove those
 test-specific patterns, here we keep the test settings commented out and only
 un-comment when running tests.
-
-First, we look for any line that matches "# [test settings]" (case insensitive,
-with optional surrounding spacing) and an optional signed integer. Any of these
-would work:
-
-    >>> assert matches('# [test settings]')
-    >>> assert matches('#[test settings]')
-    >>> assert matches('# [ test settings ]')
-    >>> assert matches('# [ test settings -1]')
-    >>> assert matches('# [ test settings +2]')
-    >>> assert matches('# [ TEST SETTINGS +2]')
-    >>> assert matches('# [ TeSt SeTTiNgS +2   ]')
-
-If a lines does not match, output it as-is.
-
-If a line matches, then uncomment it. Specifically, remove the first "#" in the
-line; if it was followed by exactly one space, then remove that too.
-
-If a line matches and a signed integer was provided, then consider it
-a relative location, and then comment-out the referred-to line. Example:
-
-    >>> preprocess('''
-    ... use this for production
-    ... # use this for tests  # [test settings -1]
-    ... '''.splitlines(True))
-    <BLANKLINE>
-    # use this for production
-    use this for tests  # [test settings -1]
-    <BLANKLINE>
-
-If the matched special string creates the first "#" in the line, then do
-nothing to that line but still respect the relative locations. Useful for just
-commenting out nearby lines for tests:
-
-    >>> preprocess('''
-    ... # [TEST SETTINGS +1]
-    ... comment out for testing'''.splitlines(True))
-    <BLANKLINE>
-    # [TEST SETTINGS +1]
-    # comment out for testing
 """
 
+
 import re
-regexp = re.compile(r'#\s?\[\s?test settings\s?(?P<rel>[-+]*\d)?\s*\]')
 
+regexp = re.compile(r"#\s?\[\s?(enable|disable) for test\s?\]")
 
-def matches(line):
-    return regexp.search(line.lower()) is not None
+
+def is_commented(line):
+    return line.strip().startswith("#")
 
 
 def comment_line(line):
@@ -66,87 +28,75 @@ def comment_line(line):
     """
     x = []
     for i, character in enumerate(line):
-        if character == ' ':
+        if character == " ":
             x.append(character)
         else:
             break
-    x.append('# ')
+    x.append("# ")
     x.extend(line[i:])
-    return ''.join(x)
+    return "".join(x)
 
 
 def uncomment_line(line):
     """
     Removes the first instance of "#" from a line; if it was followed by
-    exactly one space then remove that too.
+    exactly one space then remove that too . . . UNLESS the *only* comment is the
+    special character that triggers this behavior, in which case we do nothing.
 
     >>> assert uncomment_line('# asdf') == 'asdf'
     >>> assert uncomment_line('#asdf') == 'asdf'
     >>> assert uncomment_line('# asdf # but this should be kept') == 'asdf # but this should be kept'
     >>> assert uncomment_line('#    asdf') == '    asdf'
     >>> assert uncomment_line('  #    asdf') == '      asdf'
+    >>> assert uncomment_line('do nothing') == 'do nothing'
+    >>> assert uncomment_line('do nothing # [disable for test]') == 'do nothing # [disable for test]'
+    >>> assert uncomment_line('#uncomment # [disable for test]') == 'uncomment # [disable for test]'
     """
-    first = line.find('#')
+    first = line.find("#")
 
-    # If the first comment is the one that flag the line, then do nothing.
+    # If the first comment is the one that flagged the line, then do nothing.
     m = regexp.search(line.lower())
     if m:
         if m.start() == first:
             return line
 
-    if line[first + 1] == ' ' and line[first + 2] != ' ':
-        pattern = '# '
+    if line[first + 1] == " " and line[first + 2] != " ":
+        pattern = "# "
     else:
-        pattern = '#'
-    return line.replace(pattern, '', 1)
+        pattern = "#"
+    return line.replace(pattern, "", 1)
 
 
 def preprocess(lines):
+    result = []
 
     if isinstance(lines, str):
         lines = [lines]
 
-    # These lists will keep track of whether a line should be changed.  We need to
-    # create them ahead of time so that we can use relative indexing from line N to
-    # modify the state of lines N-1 or N+1
-    uncomment = [False for i in range(len(lines))]
-    comment = [False for i in range(len(lines))]
-
-    for i, line in enumerate(lines):
+    for line in lines:
         m = regexp.search(line.lower())
-        if m:
-            # There as at least a "[ test settings ]", so remove comment
-            uncomment[i] = True
-
-            # Figure out if there was also a relative location to uncomment,
-            # and keep track of it in the `comment` list.
-            rel = m.group('rel')
-            if rel is not None:
-                rel = int(rel)
-                comment[i + rel] = True
+        if not m:
+            result.append(line)
+            continue
 
-    result = []
-    for (c, u, line) in zip(comment, uncomment, lines):
-        # E.g., in this situation, unclear what should happen:
-        #
-        #     # [test settings]
-        #     # [test settings -1]
-        #
-        if c and u:
-            raise ValueError("Line {0} is trying to be both commented and uncommented".format(line))
-        if c:
-            result.append(comment_line(line))
-        elif u:
+        action = m.group(1)
+        if action == "enable" and is_commented(line):
             result.append(uncomment_line(line))
+        elif action == "disable" and not is_commented(line):
+            result.append(comment_line(line))
         else:
-            result.append(line)
-    print(''.join(result))
+            raise ValueError(f"Inconsistent commenting and action:\n{line}")
+
+    print("".join(result))
 
 
 if __name__ == "__main__":
     import argparse
+
     ap = argparse.ArgumentParser(usage=__doc__)
-    ap.add_argument('infile', help='Input file to modify. Modified file printed to stdout.')
+    ap.add_argument(
+        "infile", help="Input file to modify. Modified file printed to stdout."
+    )
     args = ap.parse_args()
     lines = open(args.infile).readlines()
     preprocess(lines)