Codex/update genome fasta pipe mode options

.github/workflows/test-singlem.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Run tests with Pixi
         run: |
-          pixi run -e dev --frozen pytest test
+          pixi run -e dev --frozen pytest test -v
 
   # Run after removing lock file so dependences are unlocked
   pixi_test_dependencies_optimistic:
@@ -55,4 +55,4 @@ jobs:
 
       - name: Run tests with Pixi
         run: |
-          pixi run -e dev pytest test
+          pixi run -e dev pytest test -v

admin/build_docs.py

@@ -75,6 +75,7 @@ def get_version(relpath):
                     'appraise': 'INPUT OTU TABLE OPTIONS',
                     'seqs': 'OPTIONS',
                     'metapackage': 'OPTIONS',
+                    'supplement': 'OPTIONS',
                 }
                 logging.info("For ROFF for command {}, removing everything before '{}'".format(
                     subcommand, splitters[subcommand]))

admin/environment.yml

@@ -13,7 +13,6 @@ dependencies:
 - fasttree=2.1.11
 - galah=0.4.2
 - graftm=0.15.1
-- gtdbtk=2.4.1
 - hmmer=3.2.1
 - jinja2=3.1.6
 - krona=2.8.1

admin/requirements.txt

@@ -3,7 +3,6 @@ biopython==1.85
 extern==0.4.1
 fastalite
 graftm==0.15.1
-gtdbtk==2.4.1
 Jinja2==3.1.6
 pandas==2.2.3
 pip==25.0.1

docs/AGENTS.md

@@ -0,0 +1,4 @@
+# AGENTS
+
+- Files under `docs/tools/` and `docs/advanced/` are autogenerated; do not modify them directly.
+- `docs/Installation.md` is autogenerated; make changes in `docs/Installation.md.in` instead.

docs/preludes/pipe_prelude.md

@@ -7,7 +7,9 @@ To further convert the generated taxonomic profile to other formats that might b
 
 ## Algorithm details
 
-**Details**: In its most common usage, the SingleM `pipe` subcommand takes as input raw metagenomic reads and outputs a taxonomic profile. It can also take as input whole genomes (or contigs), and can output a table of OTUs. Note that taxonomic profiles are generated from OTU tables, they are [not the same thing](/Glossary).
+**Details**: In its most common usage, the SingleM `pipe` subcommand takes as input raw metagenomic reads and outputs a taxonomic profile. Note that taxonomic profiles are generated from OTU tables, they are [not the same thing](/Glossary).
+
+It can also take as input whole genomes (or contigs) via `--genome-fasta-files`, `--genome-fasta-directory` or `--genome-fasta-list`. These options behave the same as providing sequences with `--forward`, except that different defaults for `--min-taxon-coverage` and `--min-orf-length` are used.
 
 `pipe` performs three steps:
 

docs/preludes/supplement_prelude.md

@@ -0,0 +1,16 @@
+The SingleM `supplement` subcommand adds genomes to a SingleM metapackage.
+
+**TLDR**: Supplement a metapackage with new genomes like so:
+```
+singlem supplement --new-genome-fasta-files <genome1.fna> <genome2.fna> \
+    --output-metapackage <supplemented.smpkg>
+```
+
+## GTDB-Tk
+In order to add genomes to a metapackage, their taxonomy is required. SingleM `supplement` can generate this taxonomy for new genomes using GTDB-Tk. However, GTDB-Tk is not installed by default, and so it must be installed separately.
+
+For instance, if you are using conda to manage dependencies, use:
+```
+conda install gtdbtk
+```
+Note that the version of GTDB-Tk installed must match the GTDB release of the metapackage. The [GTDB-Tk documentation](https://ecogenomics.github.io/GTDBTk/installing/index.html) provides guidance on installation and compatibility.

docs/tools/supplement.md

@@ -3,6 +3,13 @@ title: SingleM supplement
 ---
 # singlem supplement
 
+**TLDR**: Supplement a metapackage with new genomes like so:
+```
+singlem supplement --new-genome-fasta-files <genome1.fna> <genome2.fna> \
+    --output-metapackage <supplemented.smpkg>
+```
+GTDB-Tk may be needed to assign taxonomy to the new genomes. If required, install it separately (e.g. `conda install gtdbtk`) and ensure its version matches the GTDB release of the metapackage.
+
 # DESCRIPTION
 
 Create a new metapackage from a vanilla one plus new genomes

pixi.toml

@@ -11,6 +11,8 @@ scripts = ["admin/set_env_vars.sh"]
 
 [dependencies]
 python = ">=3.7"
+coreutils = "*" # So sort --parallel works
+bash = "*"
 diamond = ">=2.1.7,<2.1.11"  # 2.1.11 segfaults (on some non-test data)
 biopython = "*"
 hmmer = "*"
@@ -44,7 +46,6 @@ pyarrow = "*"
 galah = ">=0.4.0"
 sqlparse = "*"  # Required indirectly (e.g. taxtastic)
 zenodo_backpack = ">=0.3.0"
-gtdbtk = "==2.4.1" # Must match the version of GTDB release used to make the default metapackage.
 # Optional (commented out)
 # python-annoy = "*"
 # nmslib = "*"
@@ -59,6 +60,7 @@ ipython = "*"
 toml = "*"
 python-build = "*"
 setuptools-scm = "*"
+gtdbtk = "==2.4.1" # Must match the version of GTDB release used to make the default metapackage.
 
 [environments]
 dev = ["dev"]

singlem/lyrebird.py

@@ -22,7 +22,7 @@
 
 import singlem
 import singlem.pipe as pipe
-from singlem.main import add_common_pipe_arguments, add_less_common_pipe_arguments, validate_pipe_args, add_condense_arguments, generate_streaming_otu_table_from_args, get_min_orf_length, get_min_taxon_coverage
+from singlem.main import add_common_pipe_arguments, add_less_common_pipe_arguments, validate_pipe_args, add_condense_arguments, generate_streaming_otu_table_from_args, get_min_orf_length, get_min_taxon_coverage, parse_genome_fasta_files
 from singlem.pipe import SearchPipe
 from singlem.condense import Condenser
 from singlem.metapackage import LYREBIRD_DATA_ENVIRONMENT_VARIABLE, CUSTOM_TAXONOMY_DATABASE_NAME
@@ -86,6 +86,7 @@ def main():
     add_condense_arguments(condense_parser)
 
     args = bird_argparser.parse_the_args()
+    parse_genome_fasta_files(args)
 
     if args.debug:
         loglevel = logging.DEBUG
@@ -107,7 +108,6 @@ def main():
         singlem.pipe.SearchPipe().run(
             sequences = args.forward,
             reverse_read_files = args.reverse,
-            genomes = args.genome_fasta_files,
             input_sra_files = args.sra_files,
             otu_table = args.otu_table,
             archive_otu_table = args.archive_otu_table,