Develop

README.md

@@ -19,8 +19,17 @@ This package is built to either be used as a library in python projects and a co
 If you want to add auto-completion to the scripts master function [pmotools-python](scripts/pmotools-runner.py) you can add the following to your `~/.bash_completion`. This can also be found in etc/bash_completion in the current directory. Or can be generated with `pmotools-python --bash-completion`
 
 ```bash
+# bash completion for pmotools-python
+# add the below to your ~/.bash_completion
+
 _pmotools_python_complete()
 {
+    # Make sure underscores (and =) are NOT treated as word breaks
+    # so options like --pmo_files or --file=path complete as one token.
+    local _OLD_WB="${COMP_WORDBREAKS-}"
+    COMP_WORDBREAKS="${COMP_WORDBREAKS//_/}"
+    COMP_WORDBREAKS="${COMP_WORDBREAKS//=}"
+
     local cur prev
     COMPREPLY=()
     cur="${COMP_WORDS[COMP_CWORD]}"
@@ -34,6 +43,9 @@ _pmotools_python_complete()
         lines="$(${COMP_WORDS[0]} --list-plain 2>/dev/null)"
         cmds="$(printf '%s\n' "${lines}" | awk -F'\t' '{print $1}')"
         COMPREPLY=( $(compgen -W "${cmds}" -- "${cur}") )
+
+        # restore wordbreaks before returning
+        COMP_WORDBREAKS="$_OLD_WB"
         return 0
     fi
 
@@ -42,19 +54,26 @@ _pmotools_python_complete()
         local helps opts
         helps="$(${COMP_WORDS[0]} ${COMP_WORDS[1]} -h 2>/dev/null)"
         # Pull out flag tokens and split comma-separated forms
+        # Keep underscores intact in the tokens.
         opts="$(printf '%s\n' "${helps}" \
-            | sed -n 's/^[[:space:]]\{0,\}\(-[-[:alnum:]][-[:alnum:]]*\)\(, *-[[:alnum:]][-[:alnum:]]*\)\{0,\}.*/\1/p' \
+            | sed -n 's/^[[:space:]]\{0,\}\(-[-[:alnum:]_][-[:alnum:]_]*\)\(, *-[[:alnum:]_][-[:alnum:]_]*\)\{0,\}.*/\1/p' \
             | sed 's/, / /g')"
         COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+
+        COMP_WORDBREAKS="$_OLD_WB"
         return 0
     fi
 
     # 3) Otherwise, fall back to filename completion for positional args
     COMPREPLY=( $(compgen -f -- "${cur}") )
+
+    # restore original word breaks
+    COMP_WORDBREAKS="$_OLD_WB"
     return 0
 }
 
 complete -F _pmotools_python_complete pmotools-python
+
 ```
 
 ## Developer Setup

docs/conf.py

@@ -14,7 +14,7 @@
 project = "pmotools-python"
 copyright = "2024, Plasmogenepi"
 author = "Plasmogenepi"
-release = "1.0.0"
+release = "0.1.0"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

etc/bash_completion

@@ -3,6 +3,12 @@
 
 _pmotools_python_complete()
 {
+    # Make sure underscores (and =) are NOT treated as word breaks
+    # so options like --pmo_files or --file=path complete as one token.
+    local _OLD_WB="${COMP_WORDBREAKS-}"
+    COMP_WORDBREAKS="${COMP_WORDBREAKS//_/}"
+    COMP_WORDBREAKS="${COMP_WORDBREAKS//=}"
+
     local cur prev
     COMPREPLY=()
     cur="${COMP_WORDS[COMP_CWORD]}"
@@ -16,6 +22,9 @@ _pmotools_python_complete()
         lines="$(${COMP_WORDS[0]} --list-plain 2>/dev/null)"
         cmds="$(printf '%s\n' "${lines}" | awk -F'\t' '{print $1}')"
         COMPREPLY=( $(compgen -W "${cmds}" -- "${cur}") )
+
+        # restore wordbreaks before returning
+        COMP_WORDBREAKS="$_OLD_WB"
         return 0
     fi
 
@@ -24,15 +33,21 @@ _pmotools_python_complete()
         local helps opts
         helps="$(${COMP_WORDS[0]} ${COMP_WORDS[1]} -h 2>/dev/null)"
         # Pull out flag tokens and split comma-separated forms
+        # Keep underscores intact in the tokens.
         opts="$(printf '%s\n' "${helps}" \
-            | sed -n 's/^[[:space:]]\{0,\}\(-[-[:alnum:]][-[:alnum:]]*\)\(, *-[[:alnum:]][-[:alnum:]]*\)\{0,\}.*/\1/p' \
+            | sed -n 's/^[[:space:]]\{0,\}\(-[-[:alnum:]_][-[:alnum:]_]*\)\(, *-[[:alnum:]_][-[:alnum:]_]*\)\{0,\}.*/\1/p' \
             | sed 's/, / /g')"
         COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+
+        COMP_WORDBREAKS="$_OLD_WB"
         return 0
     fi
 
     # 3) Otherwise, fall back to filename completion for positional args
     COMPREPLY=( $(compgen -f -- "${cur}") )
+
+    # restore original word breaks
+    COMP_WORDBREAKS="$_OLD_WB"
     return 0
 }
 

src/pmotools/cli.py

@@ -37,7 +37,7 @@
 from pmotools.scripts.extractors_from_pmo.extract_pmo_with_read_filter import (
     extract_pmo_with_read_filter,
 )
-from pmotools.scripts.extractors_from_pmo.extract_allele_table import (
+from pmotools.scripts.pmo_to_tables.extract_allele_table import (
     extract_for_allele_table,
 )
 
@@ -66,13 +66,37 @@
 )
 
 # panel info subset
-from pmotools.scripts.extract_info_from_pmo.extract_insert_of_panels import (
+from pmotools.scripts.pmo_to_tables.extract_insert_of_panels import (
     extract_insert_of_panels,
 )
-from pmotools.scripts.extract_info_from_pmo.extract_refseq_of_inserts_of_panels import (
+from pmotools.scripts.pmo_to_tables.extract_refseq_of_inserts_of_panels import (
     extract_refseq_of_inserts_of_panels,
 )
 
+# pmo to tables
+
+from pmotools.scripts.pmo_to_tables.export_specimen_meta_table import (
+    export_specimen_meta_table,
+)
+from pmotools.scripts.pmo_to_tables.export_library_sample_meta_table import (
+    export_library_sample_meta_table,
+)
+from pmotools.scripts.pmo_to_tables.export_project_info_meta_table import (
+    export_project_info_meta_table,
+)
+from pmotools.scripts.pmo_to_tables.export_sequencing_info_meta_table import (
+    export_sequencing_info_meta_table,
+)
+from pmotools.scripts.pmo_to_tables.export_specimen_travel_meta_table import (
+    export_specimen_travel_meta_table,
+)
+from pmotools.scripts.pmo_to_tables.export_target_info_meta_table import (
+    export_target_info_meta_table,
+)
+from pmotools.scripts.pmo_to_tables.export_panel_info_meta_table import (
+    export_panel_info_meta_table,
+)
+
 
 @dataclass(frozen=True)
 class PmoCommand:
@@ -115,17 +139,6 @@ class PmoCommand:
         "extract_pmo_with_read_filter": PmoCommand(
             extract_pmo_with_read_filter, "Extract with a read filter"
         ),
-        "extract_allele_table": PmoCommand(
-            extract_for_allele_table,
-            "Extract allele tables for tools like dcifer or moire",
-        ),
-        "extract_insert_of_panels": PmoCommand(
-            extract_insert_of_panels, "Extract inserts of panels from a PMO"
-        ),
-        "extract_refseq_of_inserts_of_panels": PmoCommand(
-            extract_refseq_of_inserts_of_panels,
-            "Extract ref_seq of panel inserts from a PMO",
-        ),
     },
     "working_with_multiple_pmos": {
         "combine_pmos": PmoCommand(
@@ -160,6 +173,46 @@ class PmoCommand:
             validate_pmo, "Validate a PMO file against a JSON Schema"
         )
     },
+    "pmo_to_table": {
+        "export_specimen_meta_table": PmoCommand(
+            export_specimen_meta_table, "export the specimen meta table from a PMO file"
+        ),
+        "export_library_sample_meta_table": PmoCommand(
+            export_library_sample_meta_table,
+            "export the library_sample meta table from a PMO file",
+        ),
+        "export_project_info_meta_table": PmoCommand(
+            export_project_info_meta_table,
+            "export the project_info meta table from a PMO file",
+        ),
+        "export_sequencing_info_meta_table": PmoCommand(
+            export_sequencing_info_meta_table,
+            "export the sequencing_info meta table from a PMO file",
+        ),
+        "export_specimen_travel_meta_table": PmoCommand(
+            export_specimen_travel_meta_table,
+            "export the specimen travel_info meta table from a PMO file",
+        ),
+        "export_target_info_meta_table": PmoCommand(
+            export_target_info_meta_table,
+            "export the target info meta table from a PMO file",
+        ),
+        "export_panel_info_meta_table": PmoCommand(
+            export_panel_info_meta_table,
+            "export the panel info meta table from a PMO file",
+        ),
+        "extract_allele_table": PmoCommand(
+            extract_for_allele_table,
+            "Extract allele tables for tools like dcifer or moire",
+        ),
+        "extract_insert_of_panels": PmoCommand(
+            extract_insert_of_panels, "Extract inserts of panels from a PMO"
+        ),
+        "extract_refseq_of_inserts_of_panels": PmoCommand(
+            extract_refseq_of_inserts_of_panels,
+            "Extract ref_seq of panel inserts from a PMO",
+        ),
+    },
 }
 
 
@@ -223,6 +276,12 @@ def _print_bash_completion():
 
 _pmotools_python_complete()
 {
+    # Make sure underscores (and =) are NOT treated as word breaks
+    # so options like --pmo_files or --file=path complete as one token.
+    local _OLD_WB="${COMP_WORDBREAKS-}"
+    COMP_WORDBREAKS="${COMP_WORDBREAKS//_/}"
+    COMP_WORDBREAKS="${COMP_WORDBREAKS//=}"
+
     local cur prev
     COMPREPLY=()
     cur="${COMP_WORDS[COMP_CWORD]}"
@@ -236,6 +295,9 @@ def _print_bash_completion():
         lines="$(${COMP_WORDS[0]} --list-plain 2>/dev/null)"
         cmds="$(printf '%s\n' "${lines}" | awk -F'\t' '{print $1}')"
         COMPREPLY=( $(compgen -W "${cmds}" -- "${cur}") )
+
+        # restore wordbreaks before returning
+        COMP_WORDBREAKS="$_OLD_WB"
         return 0
     fi
 
@@ -244,19 +306,26 @@ def _print_bash_completion():
         local helps opts
         helps="$(${COMP_WORDS[0]} ${COMP_WORDS[1]} -h 2>/dev/null)"
         # Pull out flag tokens and split comma-separated forms
+        # Keep underscores intact in the tokens.
         opts="$(printf '%s\n' "${helps}" \
-            | sed -n 's/^[[:space:]]\{0,\}\(-[-[:alnum:]][-[:alnum:]]*\)\(, *-[[:alnum:]][-[:alnum:]]*\)\{0,\}.*/\1/p' \
+            | sed -n 's/^[[:space:]]\{0,\}\(-[-[:alnum:]_][-[:alnum:]_]*\)\(, *-[[:alnum:]_][-[:alnum:]_]*\)\{0,\}.*/\1/p' \
             | sed 's/, / /g')"
         COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+
+        COMP_WORDBREAKS="$_OLD_WB"
         return 0
     fi
 
     # 3) Otherwise, fall back to filename completion for positional args
     COMPREPLY=( $(compgen -f -- "${cur}") )
+
+    # restore original word breaks
+    COMP_WORDBREAKS="$_OLD_WB"
     return 0
 }
 
 complete -F _pmotools_python_complete pmotools-python
+
 """
     import sys
 

src/pmotools/pmo_builder/json_convert_utils.py

@@ -3,3 +3,48 @@ def check_additional_columns_exist(df, additional_column_list):
         missing_cols = set(additional_column_list) - set(df.columns)
         if missing_cols:
             raise ValueError(f"Missing additional columns: {missing_cols}")
+
+
+def remove_optional_null_values(json_data, optional_columns):
+    """
+    Remove empty values from optional fields in a list of dictionaries.
+
+    :param json_data: List of dictionaries to process
+    :param optional_columns: List of optional field names to check for empty values
+    :return: List of dictionaries with empty optional fields removed
+
+    Empty values include: None, empty strings (''), empty dicts ({}), and empty lists ([])
+    """
+    # Convert optional_columns to a set for faster lookup
+    optional_fields_set = set(optional_columns) if optional_columns else set()
+
+    for item in json_data:
+        # Collect keys to remove to avoid modifying dict while iterating
+        keys_to_remove = []
+        for key, value in item.items():
+            if key in optional_fields_set:
+                # Check if value is empty: None, empty string, empty dict, or empty list
+                if value is None or value == "" or value == {} or value == []:
+                    keys_to_remove.append(key)
+
+        # Remove the empty optional fields
+        for key in keys_to_remove:
+            del item[key]
+
+    return json_data
+
+
+def check_null_values(df, columns):
+    """
+    Check for null values in a list of columns
+
+    :param df: DataFrame to check
+    :param columns: List of column names to check
+    :return: None
+    """
+    null_columns = []
+    for col in columns:
+        if df[col].isna().any():
+            null_columns.append(col)
+    if null_columns:
+        raise ValueError(f"The following columns contain null values: {null_columns}")