From f62ea796f87b2d69900c52db89e792c190c792c0 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Fri, 5 Sep 2025 23:47:48 -0700 Subject: [PATCH 01/13] refactor: pass around container_settings --- spras/allpairs.py | 16 +++++++++------- spras/analysis/cytoscape.py | 12 +++++++----- spras/btb.py | 18 ++++++++++-------- spras/containers.py | 34 +++++++++++++++++----------------- spras/domino.py | 23 +++++++++++++---------- spras/meo.py | 22 ++++++++++++---------- spras/mincostflow.py | 19 ++++++++++--------- spras/omicsintegrator1.py | 18 ++++++++++-------- spras/omicsintegrator2.py | 16 +++++++++------- spras/pathlinker.py | 16 +++++++++------- spras/responsenet.py | 18 ++++++++++-------- test/test_util.py | 2 +- 12 files changed, 117 insertions(+), 97 deletions(-) diff --git a/spras/allpairs.py b/spras/allpairs.py index e0f28d748..5c2f3807d 100644 --- a/spras/allpairs.py +++ b/spras/allpairs.py @@ -1,6 +1,7 @@ import warnings from pathlib import Path +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container_and_log from spras.dataset import Dataset from spras.interactome import ( @@ -69,14 +70,15 @@ def generate_inputs(data: Dataset, filename_map): header=["#Interactor1", "Interactor2", "Weight"]) @staticmethod - def run(nodetypes=None, network=None, directed_flag=None, output_file=None, container_framework="docker"): + def run(nodetypes=None, network=None, directed_flag=None, output_file=None, container_settings=None): """ Run All Pairs Shortest Paths with Docker @param nodetypes: input node types with sources and targets (required) @param network: input network file (required) - @param container_framework: choose the container runtime framework, currently supports "docker" or "singularity" (optional) + @param container_settings: configure the container runtime @param output_file: path to the output pathway file (required) """ + if not container_settings: container_settings = ProcessedContainerSettings() if not nodetypes or not network or not output_file or not directed_flag: raise ValueError('Required All Pairs Shortest Paths arguments are missing') @@ -85,15 +87,15 @@ def run(nodetypes=None, network=None, directed_flag=None, output_file=None, cont # Each volume is a tuple (src, dest) volumes = list() - bind_path, node_file = prepare_volume(nodetypes, work_dir) + bind_path, node_file = prepare_volume(nodetypes, work_dir, container_settings) volumes.append(bind_path) - bind_path, network_file = prepare_volume(network, work_dir) + bind_path, network_file = prepare_volume(network, work_dir, container_settings) volumes.append(bind_path) # Create the parent directories for the output file if needed Path(output_file).parent.mkdir(parents=True, exist_ok=True) - bind_path, mapped_out_file = prepare_volume(output_file, work_dir) + bind_path, mapped_out_file = prepare_volume(output_file, work_dir, container_settings) volumes.append(bind_path) command = ['python', @@ -107,11 +109,11 @@ def run(nodetypes=None, network=None, directed_flag=None, output_file=None, cont container_suffix = "allpairs:v4" run_container_and_log( 'All Pairs Shortest Paths', - container_framework, container_suffix, command, volumes, - work_dir) + work_dir, + container_settings) @staticmethod def parse_output(raw_pathway_file, standardized_pathway_file, params): diff --git a/spras/analysis/cytoscape.py b/spras/analysis/cytoscape.py index 271290140..c7a69bf37 100644 --- a/spras/analysis/cytoscape.py +++ b/spras/analysis/cytoscape.py @@ -2,16 +2,18 @@ from shutil import rmtree from typing import List, Union +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container_and_log -def run_cytoscape(pathways: List[Union[str, PurePath]], output_file: str, container_framework="docker") -> None: +def run_cytoscape(pathways: List[Union[str, PurePath]], output_file: str, container_settings=None) -> None: """ Create a Cytoscape session file with visualizations of each of the provided pathways @param pathways: a list of pathways to visualize @param output_file: the output Cytoscape session file - @param container_framework: choose the container runtime framework, currently supports "docker" or "singularity" (optional) + @param container_settings: configure the container runtime """ + if not container_settings: container_settings = ProcessedContainerSettings() work_dir = '/spras' # To work with Singularity, /spras must be mapped to a writeable location because that directory is fixed as @@ -33,7 +35,7 @@ def run_cytoscape(pathways: List[Union[str, PurePath]], output_file: str, contai volumes.append((cytoscape_output_dir, PurePath(work_dir, 'CytoscapeConfiguration'))) # Map the output file - bind_path, mapped_output = prepare_volume(output_file, work_dir) + bind_path, mapped_output = prepare_volume(output_file, work_dir, container_settings) volumes.append(bind_path) # Create the initial Python command to run inside the container @@ -41,17 +43,17 @@ def run_cytoscape(pathways: List[Union[str, PurePath]], output_file: str, contai # Map the pathway filenames and add them to the Python command for pathway in pathways: - bind_path, mapped_pathway = prepare_volume(pathway, work_dir) + bind_path, mapped_pathway = prepare_volume(pathway, work_dir, container_settings) volumes.append(bind_path) # Provided the mapped pathway file path and the original file path as the label Cytoscape command.extend(['--pathway', f'{mapped_pathway}|{pathway}']) container_suffix = "py4cytoscape:v3" run_container_and_log('Cytoscape', - container_framework, container_suffix, command, volumes, work_dir, + container_settings, env) rmtree(cytoscape_output_dir) diff --git a/spras/btb.py b/spras/btb.py index ced433efe..c3694b608 100644 --- a/spras/btb.py +++ b/spras/btb.py @@ -1,5 +1,6 @@ from pathlib import Path +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container_and_log from spras.interactome import ( convert_undirected_to_directed, @@ -68,15 +69,16 @@ def generate_inputs(data, filename_map): # Skips parameter validation step @staticmethod - def run(sources=None, targets=None, edges=None, output_file=None, container_framework="docker"): + def run(sources=None, targets=None, edges=None, output_file=None, container_settings=None): """ Run BTB with Docker @param sources: input source file (required) @param targets: input target file (required) @param edges: input edge file (required) @param output_file: path to the output pathway file (required) - @param container_framework: choose the container runtime framework, currently supports "docker" or "singularity" (optional) + @param container_settings: configure the container runtime """ + if not container_settings: container_settings = ProcessedContainerSettings() # Tests for pytest (docker container also runs this) # Testing out here avoids the trouble that container errors provide @@ -105,19 +107,19 @@ def run(sources=None, targets=None, edges=None, output_file=None, container_fram # Each volume is a tuple (src, dest) volumes = list() - bind_path, source_file = prepare_volume(sources, work_dir) + bind_path, source_file = prepare_volume(sources, work_dir, container_settings) volumes.append(bind_path) - bind_path, target_file = prepare_volume(targets, work_dir) + bind_path, target_file = prepare_volume(targets, work_dir, container_settings) volumes.append(bind_path) - bind_path, edges_file = prepare_volume(edges, work_dir) + bind_path, edges_file = prepare_volume(edges, work_dir, container_settings) volumes.append(bind_path) # Use its --output argument to set the output file prefix to specify an absolute path and prefix out_dir = Path(output_file).parent out_dir.mkdir(parents=True, exist_ok=True) - bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir) + bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir, container_settings) volumes.append(bind_path) mapped_out_prefix = mapped_out_dir + '/raw-pathway.txt' # Use posix path inside the container @@ -134,11 +136,11 @@ def run(sources=None, targets=None, edges=None, output_file=None, container_fram container_suffix = "bowtiebuilder:v2" run_container_and_log('BowTieBuilder', - container_framework, container_suffix, command, volumes, - work_dir) + work_dir, + container_settings) # Output is already written to raw-pathway.txt file diff --git a/spras/containers.py b/spras/containers.py index 7fdf9f904..fb7fea766 100644 --- a/spras/containers.py +++ b/spras/containers.py @@ -8,7 +8,7 @@ import docker import docker.errors -import spras.config.config as config +from spras.config.container_schema import ProcessedContainerSettings from spras.logging import indent from spras.util import hash_filename @@ -131,47 +131,47 @@ def env_to_items(environment: dict[str, str]) -> Iterator[str]: # TODO consider a better default environment variable # Follow docker-py's naming conventions (https://docker-py.readthedocs.io/en/stable/containers.html) # Technically the argument is an image, not a container, but we use container here. -def run_container(framework: str, container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, environment: Optional[dict[str, str]] = None): +def run_container(container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, container_settings: ProcessedContainerSettings, environment: Optional[dict[str, str]] = None): """ Runs a command in the container using Singularity or Docker - @param framework: singularity or docker @param container_suffix: name of the DockerHub container without the 'docker://' prefix @param command: command to run in the container @param volumes: a list of volumes to mount where each item is a (source, destination) tuple @param working_dir: the working directory in the container + @param container_settings: the settings to use to run the container @param environment: environment variables to set in the container @return: output from Singularity execute or Docker run """ - normalized_framework = framework.casefold() + normalized_framework = container_settings.framework.casefold() - container = config.config.container_settings.prefix + "/" + container_suffix + container = container_settings.prefix + "/" + container_suffix if normalized_framework == 'docker': return run_container_docker(container, command, volumes, working_dir, environment) elif normalized_framework == 'singularity': - return run_container_singularity(container, command, volumes, working_dir, environment) + return run_container_singularity(container, command, volumes, working_dir, container_settings, environment) elif normalized_framework == 'dsub': return run_container_dsub(container, command, volumes, working_dir, environment) else: - raise ValueError(f'{framework} is not a recognized container framework. Choose "docker", "dsub", or "singularity".') + raise ValueError(f'{container_settings.framework} is not a recognized container framework. Choose "docker", "dsub", or "singularity".') -def run_container_and_log(name: str, framework: str, container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, environment: Optional[dict[str, str]] = None): +def run_container_and_log(name: str, container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, container_settings: ProcessedContainerSettings, environment: Optional[dict[str, str]] = None): """ Runs a command in the container using Singularity or Docker with associated pretty printed messages. @param name: the display name of the running container for logging purposes - @param framework: singularity or docker @param container_suffix: name of the DockerHub container without the 'docker://' prefix @param command: command to run in the container @param volumes: a list of volumes to mount where each item is a (source, destination) tuple @param working_dir: the working directory in the container + @param container_settings: the container settings to use @param environment: environment variables to set in the container @return: output from Singularity execute or Docker run """ if not environment: environment = {'SPRAS': 'True'} - print('Running {} on container framework "{}" on env {} with command: {}'.format(name, framework, list(env_to_items(environment)), ' '.join(command)), flush=True) + print('Running {} on container framework "{}" on env {} with command: {}'.format(name, container_settings.framework, list(env_to_items(environment)), ' '.join(command)), flush=True) try: - out = run_container(framework=framework, container_suffix=container_suffix, command=command, volumes=volumes, working_dir=working_dir, environment=environment) + out = run_container(container_suffix=container_suffix, command=command, volumes=volumes, working_dir=working_dir, container_settings=container_settings, environment=environment) if out is not None: if isinstance(out, list): out = ''.join(out) @@ -290,7 +290,7 @@ def run_container_docker(container: str, command: List[str], volumes: List[Tuple return out -def run_container_singularity(container: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, environment: Optional[dict[str, str]] = None): +def run_container_singularity(container: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, config: ProcessedContainerSettings, environment: Optional[dict[str, str]] = None): """ Runs a command in the container using Singularity. Only available on Linux. @@ -329,7 +329,7 @@ def run_container_singularity(container: str, command: List[str], volumes: List[ singularity_options.extend(['--env', ",".join(env_to_items(environment))]) # Handle unpacking singularity image if needed. Potentially needed for running nested unprivileged containers - if config.config.container_settings.unpack_singularity: + if config.unpack_singularity: # Split the string by "/" path_elements = container.split("/") @@ -369,7 +369,7 @@ def run_container_singularity(container: str, command: List[str], volumes: List[ # Because this is called independently for each file, the same local path can be mounted to multiple volumes -def prepare_volume(filename: Union[str, PurePath], volume_base: Union[str, PurePath]) -> Tuple[Tuple[PurePath, PurePath], str]: +def prepare_volume(filename: Union[str, os.PathLike], volume_base: Union[str, PurePath], config: ProcessedContainerSettings) -> Tuple[Tuple[PurePath, PurePath], str]: """ Makes a file on the local file system accessible within a container by mapping the local (source) path to a new container (destination) path and renaming the file to be relative to the destination path. @@ -385,10 +385,10 @@ def prepare_volume(filename: Union[str, PurePath], volume_base: Union[str, PureP if not base_path.is_absolute(): raise ValueError(f'Volume base must be an absolute path: {volume_base}') - if isinstance(filename, PurePath): + if isinstance(filename, os.PathLike): filename = str(filename) - filename_hash = hash_filename(filename, config.config.container_settings.hash_length) + filename_hash = hash_filename(filename, config.hash_length) dest = PurePosixPath(base_path, filename_hash) abs_filename = Path(filename).resolve() @@ -477,4 +477,4 @@ def run_container_dsub(container: str, command: List[str], volumes: List[Tuple[P download_gcs(local_path=str(src), gcs_path=gcs_path, is_dir=True) # return location of dsub logs in WORKSPACE_BUCKET - return 'dsub logs: {logs}'.format(logs=flags['logging']) + return 'dsub logs: {logs}'.format(logs=flags['logging']) \ No newline at end of file diff --git a/spras/domino.py b/spras/domino.py index 449715b1f..17c8da0ad 100644 --- a/spras/domino.py +++ b/spras/domino.py @@ -3,6 +3,8 @@ import pandas as pd + +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container_and_log from spras.interactome import ( add_constant, @@ -70,7 +72,7 @@ def generate_inputs(data, filename_map): header=['ID_interactor_A', 'ppi', 'ID_interactor_B']) @staticmethod - def run(network=None, active_genes=None, output_file=None, slice_threshold=None, module_threshold=None, container_framework="docker"): + def run(network=None, active_genes=None, output_file=None, slice_threshold=None, module_threshold=None, container_settings=None): """ Run DOMINO with Docker. Let visualization be always true, parallelization be always 1 thread, and use_cache be always false. @@ -80,8 +82,9 @@ def run(network=None, active_genes=None, output_file=None, slice_threshold=None, @param output_file: path to the output pathway file (required) @param slice_threshold: the p-value threshold for considering a slice as relevant (optional) @param module_threshold: the p-value threshold for considering a putative module as final module (optional) - @param container_framework: choose the container runtime framework, currently supports "docker" or "singularity" (optional) + @param container_settings: configure the container runtime """ + if not container_settings: container_settings = ProcessedContainerSettings() if not network or not active_genes or not output_file: raise ValueError('Required DOMINO arguments are missing') @@ -91,19 +94,19 @@ def run(network=None, active_genes=None, output_file=None, slice_threshold=None, # Each volume is a tuple (source, destination) volumes = list() - bind_path, network_file = prepare_volume(network, work_dir) + bind_path, network_file = prepare_volume(network, work_dir, container_settings) volumes.append(bind_path) - bind_path, node_file = prepare_volume(active_genes, work_dir) + bind_path, node_file = prepare_volume(active_genes, work_dir, container_settings) volumes.append(bind_path) out_dir = Path(output_file).parent out_dir.mkdir(parents=True, exist_ok=True) - bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir) + bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir, container_settings) volumes.append(bind_path) slices_file = Path(out_dir, 'slices.txt') - bind_path, mapped_slices_file = prepare_volume(str(slices_file), work_dir) + bind_path, mapped_slices_file = prepare_volume(str(slices_file), work_dir, container_settings) volumes.append(bind_path) # Make the Python command to run within the container @@ -113,11 +116,11 @@ def run(network=None, active_genes=None, output_file=None, slice_threshold=None, container_suffix = "domino" run_container_and_log('slicer', - container_framework, container_suffix, slicer_command, volumes, - work_dir) + work_dir, + container_settings) # Make the Python command to run within the container domino_command = ['domino', @@ -137,11 +140,11 @@ def run(network=None, active_genes=None, output_file=None, slice_threshold=None, domino_command.extend(['--module_threshold', str(module_threshold)]) run_container_and_log('DOMINO', - container_framework, container_suffix, domino_command, volumes, - work_dir) + work_dir, + container_settings) # DOMINO creates a new folder in out_dir to output its modules HTML files into called active_genes # The filename is determined by the input active_genes and cannot be configured diff --git a/spras/meo.py b/spras/meo.py index 3ae8be9cc..9461558ef 100644 --- a/spras/meo.py +++ b/spras/meo.py @@ -1,6 +1,7 @@ import os from pathlib import Path +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container_and_log from spras.interactome import ( add_directionality_constant, @@ -125,7 +126,7 @@ def generate_inputs(data, filename_map): # TODO document required arguments @staticmethod def run(edges=None, sources=None, targets=None, output_file=None, max_path_length=None, local_search=None, - rand_restarts=None, container_framework="docker"): + rand_restarts=None, container_settings=None): """ Run Maximum Edge Orientation in the Docker image with the provided parameters. The properties file is generated from the provided arguments. @@ -137,8 +138,9 @@ def run(edges=None, sources=None, targets=None, output_file=None, max_path_lengt @param max_path_length: the maximal length of a path from sources and targets to orient. @param local_search: a "Yes"/"No" parameter that enables MEO's local search functionality. See "Improving approximations with local search" in the associated paper for more information. @param rand_restarts: The (int) of random restarts to use. - @param container_framework: choose the container runtime framework, currently supports "docker" or "singularity" (optional) + @param container_settings: configure the container runtime """ + if not container_settings: container_settings = ProcessedContainerSettings() if edges is None or sources is None or targets is None or output_file is None: raise ValueError('Required Maximum Edge Orientation arguments are missing') @@ -147,25 +149,25 @@ def run(edges=None, sources=None, targets=None, output_file=None, max_path_lengt # Each volume is a tuple (src, dest) volumes = list() - bind_path, edge_file = prepare_volume(edges, work_dir) + bind_path, edge_file = prepare_volume(edges, work_dir, container_settings) volumes.append(bind_path) - bind_path, source_file = prepare_volume(sources, work_dir) + bind_path, source_file = prepare_volume(sources, work_dir, container_settings) volumes.append(bind_path) - bind_path, target_file = prepare_volume(targets, work_dir) + bind_path, target_file = prepare_volume(targets, work_dir, container_settings) volumes.append(bind_path) out_dir = Path(output_file).parent # Maximum Edge Orientation requires that the output directory exist out_dir.mkdir(parents=True, exist_ok=True) - bind_path, mapped_output_file = prepare_volume(str(output_file), work_dir) + bind_path, mapped_output_file = prepare_volume(str(output_file), work_dir, container_settings) volumes.append(bind_path) # Hard code the path output filename, which will be deleted path_output_file = Path(out_dir, 'path-output.txt') - bind_path, mapped_path_output = prepare_volume(str(path_output_file), work_dir) + bind_path, mapped_path_output = prepare_volume(str(path_output_file), work_dir, container_settings) volumes.append(bind_path) properties_file = 'meo-properties.txt' @@ -173,7 +175,7 @@ def run(edges=None, sources=None, targets=None, output_file=None, max_path_lengt write_properties(filename=properties_file_local, edges=edge_file, sources=source_file, targets=target_file, edge_output=mapped_output_file, path_output=mapped_path_output, max_path_length=max_path_length, local_search=local_search, rand_restarts=rand_restarts, framework=container_framework) - bind_path, properties_file = prepare_volume(str(properties_file_local), work_dir) + bind_path, properties_file = prepare_volume(str(properties_file_local), work_dir, container_settings) volumes.append(bind_path) command = ['java', '-jar', '/meo/EOMain.jar', properties_file] @@ -181,10 +183,10 @@ def run(edges=None, sources=None, targets=None, output_file=None, max_path_lengt container_suffix = "meo" run_container_and_log('Maximum Edge Orientation', container_framework, - container_suffix, command, volumes, - work_dir) + work_dir, + container_settings) properties_file_local.unlink(missing_ok=True) diff --git a/spras/mincostflow.py b/spras/mincostflow.py index 03898a1bd..d77089f2f 100644 --- a/spras/mincostflow.py +++ b/spras/mincostflow.py @@ -1,5 +1,6 @@ from pathlib import Path +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container_and_log from spras.interactome import ( convert_undirected_to_directed, @@ -60,7 +61,7 @@ def generate_inputs(data, filename_map): header=False) @staticmethod - def run(sources=None, targets=None, edges=None, output_file=None, flow=None, capacity=None, container_framework="docker"): + def run(sources=None, targets=None, edges=None, output_file=None, flow=None, capacity=None, container_settings=None): """ Run min cost flow with Docker (or singularity) @param sources: input sources (required) @@ -69,9 +70,9 @@ def run(sources=None, targets=None, edges=None, output_file=None, flow=None, cap @param output_file: output file name (required) @param flow: (int) amount of flow going through the graph (optional) @param capacity: (float) amount of capacity allowed on each edge (optional) - @param container_framework: choose the container runtime framework, currently supports "docker" or "singularity" (optional) + @param container_settings: configure the container runtime """ - + if not container_settings: container_settings = ProcessedContainerSettings() # ensures that these parameters are required if not sources or not targets or not edges or not output_file: raise ValueError('Required MinCostFlow arguments are missing') @@ -82,19 +83,19 @@ def run(sources=None, targets=None, edges=None, output_file=None, flow=None, cap # the tuple is for mapping the sources, targets, edges, and output volumes = list() - bind_path, sources_file = prepare_volume(sources, work_dir) + bind_path, sources_file = prepare_volume(sources, work_dir, container_settings) volumes.append(bind_path) - bind_path, targets_file = prepare_volume(targets, work_dir) + bind_path, targets_file = prepare_volume(targets, work_dir, container_settings) volumes.append(bind_path) - bind_path, edges_file = prepare_volume(edges, work_dir) + bind_path, edges_file = prepare_volume(edges, work_dir, container_settings) volumes.append(bind_path) # Create a prefix for the output filename and ensure the directory exists out_dir = Path(output_file).parent out_dir.mkdir(parents=True, exist_ok=True) - bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir) + bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir, container_settings) volumes.append(bind_path) mapped_out_prefix = mapped_out_dir + '/out' @@ -117,11 +118,11 @@ def run(sources=None, targets=None, edges=None, output_file=None, flow=None, cap # constructs a docker run call run_container_and_log('MinCostFlow', - container_framework, container_suffix, command, volumes, - work_dir) + work_dir, + container_settings) # Check the output of the container out_dir_content = sorted(out_dir.glob('*.sif')) diff --git a/spras/omicsintegrator1.py b/spras/omicsintegrator1.py index c0b764276..c9eb0ba2f 100644 --- a/spras/omicsintegrator1.py +++ b/spras/omicsintegrator1.py @@ -1,5 +1,6 @@ from pathlib import Path +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container_and_log from spras.interactome import reinsert_direction_col_mixed from spras.prm import PRM @@ -100,7 +101,7 @@ def generate_inputs(data, filename_map): @staticmethod def run(edges=None, prizes=None, dummy_nodes=None, dummy_mode=None, mu_squared=None, exclude_terms=None, output_file=None, noisy_edges=None, shuffled_prizes=None, random_terminals=None, - seed=None, w=None, b=None, d=None, mu=None, noise=None, g=None, r=None, container_framework="docker"): + seed=None, w=None, b=None, d=None, mu=None, noise=None, g=None, r=None, container_settings=None): """ Run Omics Integrator 1 in the Docker image with the provided parameters. Does not support the garnet, cyto30, knockout, cv, or cv-reps arguments. @@ -122,8 +123,9 @@ def run(edges=None, prizes=None, dummy_nodes=None, dummy_mode=None, mu_squared=N @param noise: Standard Deviation of the gaussian noise added to edges in Noisy Edges Randomizations @param g: (gamma) msgsteiner reinforcement parameter that affects the convergence of the solution and runtime, with larger values leading to faster convergence but suboptimal results (default 0.001) @param r: msgsteiner parameter that adds random noise to edges, which is rarely needed (default 0) - @param container_framework: choose the container runtime framework, currently supports "docker" or "singularity" (optional) + @param container_settings: configure the container runtime """ + if not container_settings: container_settings = ProcessedContainerSettings() if edges is None or prizes is None or output_file is None or w is None or b is None or d is None: raise ValueError('Required Omics Integrator 1 arguments are missing') @@ -132,10 +134,10 @@ def run(edges=None, prizes=None, dummy_nodes=None, dummy_mode=None, mu_squared=N # Each volume is a tuple (src, dest) volumes = list() - bind_path, edge_file = prepare_volume(edges, work_dir) + bind_path, edge_file = prepare_volume(edges, work_dir, container_settings) volumes.append(bind_path) - bind_path, prize_file = prepare_volume(prizes, work_dir) + bind_path, prize_file = prepare_volume(prizes, work_dir, container_settings) volumes.append(bind_path) # 4 dummy mode possibilities: @@ -148,20 +150,20 @@ def run(edges=None, prizes=None, dummy_nodes=None, dummy_mode=None, mu_squared=N if dummy_mode == 'file': if dummy_nodes is None: raise ValueError("dummy_nodes file is required when dummy_mode is set to 'file'") - bind_path, dummy_file = prepare_volume(dummy_nodes, work_dir) + bind_path, dummy_file = prepare_volume(dummy_nodes, work_dir, container_settings) volumes.append(bind_path) out_dir = Path(output_file).parent # Omics Integrator 1 requires that the output directory exist out_dir.mkdir(parents=True, exist_ok=True) - bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir) + bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir, container_settings) volumes.append(bind_path) conf_file = 'oi1-configuration.txt' conf_file_local = Path(out_dir, conf_file) # Temporary file that will be deleted after running Omics Integrator 1 write_conf(conf_file_local, w=w, b=b, d=d, mu=mu, noise=noise, g=g, r=r) - bind_path, conf_file = prepare_volume(str(conf_file_local), work_dir) + bind_path, conf_file = prepare_volume(str(conf_file_local), work_dir, container_settings) volumes.append(bind_path) command = ['python', '/OmicsIntegrator/scripts/forest.py', @@ -197,11 +199,11 @@ def run(edges=None, prizes=None, dummy_nodes=None, dummy_mode=None, mu_squared=N container_suffix = "omics-integrator-1:no-conda" # no-conda version is the default run_container_and_log('Omics Integrator 1', - container_framework, container_suffix, # no-conda version is the default command, volumes, work_dir, + container_settings, {'TMPDIR': mapped_out_dir}) conf_file_local.unlink(missing_ok=True) diff --git a/spras/omicsintegrator2.py b/spras/omicsintegrator2.py index ee133f2bd..8ad680f48 100644 --- a/spras/omicsintegrator2.py +++ b/spras/omicsintegrator2.py @@ -2,6 +2,7 @@ import pandas as pd +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container_and_log from spras.dataset import Dataset from spras.interactome import reinsert_direction_col_undirected @@ -67,7 +68,7 @@ def generate_inputs(data: Dataset, filename_map): # TODO add reasonable default values @staticmethod def run(edges=None, prizes=None, output_file=None, w=None, b=None, g=None, noise=None, noisy_edges=None, - random_terminals=None, dummy_mode=None, seed=None, container_framework="docker"): + random_terminals=None, dummy_mode=None, seed=None, container_settings=None): """ Run Omics Integrator 2 in the Docker image with the provided parameters. Only the .tsv output file is retained and then renamed. @@ -84,8 +85,9 @@ def run(edges=None, prizes=None, output_file=None, w=None, b=None, g=None, noise "others" = connect to all nodes except for terminals "all" = connect to all nodes in the interactome. @param seed: The random seed to use for this run. - @param container_framework: choose the container runtime framework, currently supports "docker" or "singularity" (optional) + @param container_settings: configure the container runtime """ + if not container_settings: container_settings = ProcessedContainerSettings() if edges is None or prizes is None or output_file is None: raise ValueError('Required Omics Integrator 2 arguments are missing') @@ -94,16 +96,16 @@ def run(edges=None, prizes=None, output_file=None, w=None, b=None, g=None, noise # Each volume is a tuple (src, dest) volumes = list() - bind_path, edge_file = prepare_volume(edges, work_dir) + bind_path, edge_file = prepare_volume(edges, work_dir, container_settings) volumes.append(bind_path) - bind_path, prize_file = prepare_volume(prizes, work_dir) + bind_path, prize_file = prepare_volume(prizes, work_dir, container_settings) volumes.append(bind_path) out_dir = Path(output_file).parent # Omics Integrator 2 requires that the output directory exist out_dir.mkdir(parents=True, exist_ok=True) - bind_path, mapped_out_dir = prepare_volume(out_dir, work_dir) + bind_path, mapped_out_dir = prepare_volume(out_dir, work_dir, container_settings) volumes.append(bind_path) command = ['OmicsIntegrator', '-e', edge_file, '-p', prize_file, @@ -130,11 +132,11 @@ def run(edges=None, prizes=None, output_file=None, w=None, b=None, g=None, noise container_suffix = "omics-integrator-2:v2" run_container_and_log('Omics Integrator 2', - container_framework, container_suffix, command, volumes, - work_dir) + work_dir, + container_settings) # TODO do we want to retain other output files? # TODO if deleting other output files, write them all to a tmp directory and copy diff --git a/spras/pathlinker.py b/spras/pathlinker.py index a671c9b92..9de26d1f5 100644 --- a/spras/pathlinker.py +++ b/spras/pathlinker.py @@ -1,6 +1,7 @@ import warnings from pathlib import Path +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container_and_log from spras.dataset import Dataset from spras.interactome import ( @@ -66,15 +67,16 @@ def generate_inputs(data, filename_map): # Skips parameter validation step @staticmethod - def run(nodetypes=None, network=None, output_file=None, k=None, container_framework="docker"): + def run(nodetypes=None, network=None, output_file=None, k=None, container_settings=None): """ Run PathLinker with Docker @param nodetypes: input node types with sources and targets (required) @param network: input network file (required) @param output_file: path to the output pathway file (required) @param k: path length (optional) - @param container_framework: choose the container runtime framework, currently supports "docker" or "singularity" (optional) + @param container_settings: configure the container runtime """ + if not container_settings: container_settings = ProcessedContainerSettings() # Add additional parameter validation # Do not require k # Use the PathLinker default @@ -87,10 +89,10 @@ def run(nodetypes=None, network=None, output_file=None, k=None, container_framew # Each volume is a tuple (src, dest) volumes = list() - bind_path, node_file = prepare_volume(nodetypes, work_dir) + bind_path, node_file = prepare_volume(nodetypes, work_dir, container_settings) volumes.append(bind_path) - bind_path, network_file = prepare_volume(network, work_dir) + bind_path, network_file = prepare_volume(network, work_dir, container_settings) volumes.append(bind_path) # PathLinker does not provide an argument to set the output directory @@ -98,7 +100,7 @@ def run(nodetypes=None, network=None, output_file=None, k=None, container_framew out_dir = Path(output_file).parent # PathLinker requires that the output directory exist out_dir.mkdir(parents=True, exist_ok=True) - bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir) + bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir, container_settings) volumes.append(bind_path) mapped_out_prefix = mapped_out_dir + '/out' # Use posix path inside the container @@ -114,11 +116,11 @@ def run(nodetypes=None, network=None, output_file=None, k=None, container_framew container_suffix = "pathlinker:v2" run_container_and_log('PathLinker', - container_framework, container_suffix, command, volumes, - work_dir) + work_dir, + container_settings) # Rename the primary output file to match the desired output filename # Currently PathLinker only writes one output file so we do not need to delete others diff --git a/spras/responsenet.py b/spras/responsenet.py index 48dd269cc..86df99869 100644 --- a/spras/responsenet.py +++ b/spras/responsenet.py @@ -1,5 +1,6 @@ from pathlib import Path +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container_and_log from spras.interactome import ( convert_undirected_to_directed, @@ -53,7 +54,7 @@ def generate_inputs(data, filename_map): header=False) @staticmethod - def run(sources=None, targets=None, edges=None, output_file=None, gamma=10, container_framework="docker"): + def run(sources=None, targets=None, edges=None, output_file=None, gamma=10, container_settings=None): """ Run ResponseNet with Docker (or singularity) @param sources: input sources (required) @@ -61,8 +62,9 @@ def run(sources=None, targets=None, edges=None, output_file=None, gamma=10, cont @param edges: input network file (required) @param output_file: output file name (required) @param gamma: integer representing gamma (optional, default is 10) - @param container_framework: choose the container runtime framework, currently supports "docker" or "singularity" (optional) + @param container_settings: configure the container runtime """ + if not container_settings: container_settings = ProcessedContainerSettings() # ensures that these parameters are required if not sources or not targets or not edges or not output_file: @@ -74,19 +76,19 @@ def run(sources=None, targets=None, edges=None, output_file=None, gamma=10, cont # the tuple is for mapping the sources, targets, edges, and output volumes = list() - bind_path, sources_file = prepare_volume(sources, work_dir) + bind_path, sources_file = prepare_volume(sources, work_dir, container_settings) volumes.append(bind_path) - bind_path, targets_file = prepare_volume(targets, work_dir) + bind_path, targets_file = prepare_volume(targets, work_dir, container_settings) volumes.append(bind_path) - bind_path, edges_file = prepare_volume(edges, work_dir) + bind_path, edges_file = prepare_volume(edges, work_dir, container_settings) volumes.append(bind_path) # Create a prefix for the output filename and ensure the directory exists out_dir = Path(output_file).parent out_dir.mkdir(parents=True, exist_ok=True) - bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir) + bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir, container_settings) volumes.append(bind_path) mapped_out_prefix = Path(mapped_out_dir) @@ -106,11 +108,11 @@ def run(sources=None, targets=None, edges=None, output_file=None, gamma=10, cont # constructs a docker run call run_container_and_log('ResponseNet', - container_framework, container_suffix, command, volumes, - work_dir) + work_dir, + container_settings) # Rename the primary output file to match the desired output filename out_file_suffixed.rename(output_file) diff --git a/test/test_util.py b/test/test_util.py index 2a25fc0d1..c18a35f75 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -41,7 +41,7 @@ def test_hash_params_sha1_base32(self): ('test/OmicsIntegrator1/output', PurePosixPath('/spras'), '/spras/TNDO5TR/output'), ('../src', '/spras', '/spras/NNBVZ6X/src')]) def test_prepare_volume(self, filename, volume_base, expected_filename): - _, container_filename = prepare_volume(filename, volume_base) + _, container_filename = prepare_volume(filename, volume_base, config.config.container_settings) assert container_filename == expected_filename def test_convert_docker_path(self): From d9d38bfa514f5c0bfcbbee04086f9e9093f86412 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Sat, 6 Sep 2025 07:00:54 +0000 Subject: [PATCH 02/13] test: correct all container settings --- spras/rwr.py | 16 +++++++++------- spras/strwr.py | 18 ++++++++++-------- test/AllPairs/test_ap.py | 5 +++-- test/BowTieBuilder/test_btb.py | 3 ++- test/DOMINO/test_domino.py | 3 ++- test/MEO/test_meo.py | 3 ++- test/MinCostFlow/test_mcf.py | 3 ++- test/OmicsIntegrator1/test_oi1.py | 3 ++- test/OmicsIntegrator2/test_oi2.py | 3 ++- test/PathLinker/test_pathlinker.py | 3 ++- test/RWR/test_RWR.py | 3 ++- test/ResponseNet/test_rn.py | 4 +++- test/ST_RWR/test_STRWR.py | 3 ++- 13 files changed, 43 insertions(+), 27 deletions(-) diff --git a/spras/rwr.py b/spras/rwr.py index 396cf4df1..156da5646 100644 --- a/spras/rwr.py +++ b/spras/rwr.py @@ -2,6 +2,7 @@ import pandas as pd +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container from spras.dataset import Dataset from spras.interactome import reinsert_direction_col_directed @@ -32,7 +33,8 @@ def generate_inputs(data, filename_map): edges.to_csv(filename_map['network'],sep='|',index=False,columns=['Interactor1','Interactor2'],header=False) @staticmethod - def run(network=None, nodes=None, alpha=None, output_file=None, container_framework="docker", threshold=None): + def run(network=None, nodes=None, alpha=None, output_file=None, container_settings=None, threshold=None): + if not container_settings: container_settings = ProcessedContainerSettings() if not nodes: raise ValueError('Required RWR arguments are missing') @@ -47,10 +49,10 @@ def run(network=None, nodes=None, alpha=None, output_file=None, container_framew # Each volume is a tuple (src, dest) volumes = list() - bind_path, nodes_file = prepare_volume(nodes, work_dir) + bind_path, nodes_file = prepare_volume(nodes, work_dir, container_settings) volumes.append(bind_path) - bind_path, network_file = prepare_volume(network, work_dir) + bind_path, network_file = prepare_volume(network, work_dir, container_settings) volumes.append(bind_path) # RWR does not provide an argument to set the output directory @@ -58,7 +60,7 @@ def run(network=None, nodes=None, alpha=None, output_file=None, container_framew out_dir = Path(output_file).parent # RWR requires that the output directory exist out_dir.mkdir(parents=True, exist_ok=True) - bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir) + bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir, container_settings) volumes.append(bind_path) mapped_out_prefix = mapped_out_dir + "/output.txt" command = ['python', @@ -72,11 +74,11 @@ def run(network=None, nodes=None, alpha=None, output_file=None, container_framew command.extend(['--alpha', str(alpha)]) container_suffix = 'rwr:v1' - out = run_container(container_framework, - container_suffix, + out = run_container(container_suffix, command, volumes, - work_dir) + work_dir, + container_settings) print(out) # Rename the primary output file to match the desired output filename diff --git a/spras/strwr.py b/spras/strwr.py index c4e1df95c..c57385009 100644 --- a/spras/strwr.py +++ b/spras/strwr.py @@ -1,5 +1,6 @@ from pathlib import Path +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import prepare_volume, run_container from spras.dataset import Dataset from spras.interactome import reinsert_direction_col_directed @@ -32,7 +33,8 @@ def generate_inputs(data, filename_map): edges.to_csv(filename_map['network'],sep='|',index=False,columns=['Interactor1','Interactor2'],header=False) @staticmethod - def run(network=None, sources=None, targets=None, alpha=None, output_file=None, container_framework="docker", threshold=None): + def run(network=None, sources=None, targets=None, alpha=None, output_file=None, container_settings=None, threshold=None): + if not container_settings: container_settings = ProcessedContainerSettings() if not sources or not targets or not network or not output_file: raise ValueError('Required local_neighborhood arguments are missing') @@ -48,13 +50,13 @@ def run(network=None, sources=None, targets=None, alpha=None, output_file=None, # Each volume is a tuple (src, dest) volumes = list() - bind_path, source_file = prepare_volume(sources, work_dir) + bind_path, source_file = prepare_volume(sources, work_dir, container_settings) volumes.append(bind_path) - bind_path, target_file = prepare_volume(targets, work_dir) + bind_path, target_file = prepare_volume(targets, work_dir, container_settings) volumes.append(bind_path) - bind_path, network_file = prepare_volume(network, work_dir) + bind_path, network_file = prepare_volume(network, work_dir, container_settings) volumes.append(bind_path) # ST_RWR does not provide an argument to set the output directory @@ -62,7 +64,7 @@ def run(network=None, sources=None, targets=None, alpha=None, output_file=None, out_dir = Path(output_file).parent # ST_RWR requires that the output directory exist out_dir.mkdir(parents=True, exist_ok=True) - bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir) + bind_path, mapped_out_dir = prepare_volume(str(out_dir), work_dir, container_settings) volumes.append(bind_path) mapped_out_prefix = mapped_out_dir + "/output.txt" command = ['python', @@ -77,11 +79,11 @@ def run(network=None, sources=None, targets=None, alpha=None, output_file=None, command.extend(['--alpha', str(alpha)]) container_suffix = 'st-rwr:v1' - out = run_container(container_framework, - container_suffix, + out = run_container(container_suffix, command, volumes, - work_dir) + work_dir, + container_settings) print(out) # Rename the primary output file to match the desired output filename diff --git a/test/AllPairs/test_ap.py b/test/AllPairs/test_ap.py index 355033f9f..e4c921f37 100644 --- a/test/AllPairs/test_ap.py +++ b/test/AllPairs/test_ap.py @@ -4,6 +4,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.allpairs import AllPairs @@ -73,7 +74,7 @@ def test_allpairs_singularity(self): network=str(TEST_DIR / 'input' / 'sample-in-net.txt'), directed_flag=str(TEST_DIR / 'input' / 'directed-flag-false.txt'), output_file=str(out_path), - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) assert out_path.exists() @pytest.mark.skipif(not shutil.which('singularity'), reason='Singularity not found on system') @@ -87,7 +88,7 @@ def test_allpairs_singularity_unpacked(self): network=str(TEST_DIR / 'input/sample-in-net.txt'), directed_flag=str(TEST_DIR / 'input' / 'directed-flag-false.txt'), output_file=str(out_path), - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) config.config.container_settings.unpack_singularity = False assert out_path.exists() diff --git a/test/BowTieBuilder/test_btb.py b/test/BowTieBuilder/test_btb.py index 8b7997692..dfb728e2e 100644 --- a/test/BowTieBuilder/test_btb.py +++ b/test/BowTieBuilder/test_btb.py @@ -4,6 +4,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config config.init_from_file("config/config.yaml") @@ -317,4 +318,4 @@ def test_btb_singularity(self): sources=Path(TEST_DIR, 'input', 'btb-sources.txt'), targets=Path(TEST_DIR, 'input', 'btb-targets.txt'), output_file=OUT_FILE_DEFAULT, - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) diff --git a/test/DOMINO/test_domino.py b/test/DOMINO/test_domino.py index 05bd3ae70..e4cb899b8 100644 --- a/test/DOMINO/test_domino.py +++ b/test/DOMINO/test_domino.py @@ -3,6 +3,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.domino import DOMINO, post_domino_id_transform, pre_domino_id_transform @@ -74,7 +75,7 @@ def test_domino_singularity(self): network=TEST_DIR+'input/domino-network.txt', active_genes=TEST_DIR+'input/domino-active-genes.txt', output_file=OUT_FILE_DEFAULT, - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) assert out_path.exists() def test_pre_id_transform(self): diff --git a/test/MEO/test_meo.py b/test/MEO/test_meo.py index 32958be20..5bc564b39 100644 --- a/test/MEO/test_meo.py +++ b/test/MEO/test_meo.py @@ -3,6 +3,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.meo import MEO, write_properties @@ -66,5 +67,5 @@ def test_meo_singularity(self): sources=TEST_DIR + 'input/meo-sources.txt', targets=TEST_DIR + 'input/meo-targets.txt', output_file=OUT_FILE, - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) assert out_path.exists() diff --git a/test/MinCostFlow/test_mcf.py b/test/MinCostFlow/test_mcf.py index c777a665d..ee9f83e98 100644 --- a/test/MinCostFlow/test_mcf.py +++ b/test/MinCostFlow/test_mcf.py @@ -3,6 +3,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.mincostflow import MinCostFlow @@ -112,6 +113,6 @@ def test_mincostflow_singularity(self, graph): output_file=OUT_FILE, flow=1, capacity=1, - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) assert out_path.exists() diff --git a/test/OmicsIntegrator1/test_oi1.py b/test/OmicsIntegrator1/test_oi1.py index a484c0af3..5e59314b9 100644 --- a/test/OmicsIntegrator1/test_oi1.py +++ b/test/OmicsIntegrator1/test_oi1.py @@ -3,6 +3,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.omicsintegrator1 import OmicsIntegrator1, write_conf @@ -124,5 +125,5 @@ def test_oi1_singularity(self): w=5, b=1, d=10, - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) assert out_path.exists() diff --git a/test/OmicsIntegrator2/test_oi2.py b/test/OmicsIntegrator2/test_oi2.py index aa74cd94e..d8310de5e 100644 --- a/test/OmicsIntegrator2/test_oi2.py +++ b/test/OmicsIntegrator2/test_oi2.py @@ -3,6 +3,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.omicsintegrator2 import OmicsIntegrator2 @@ -67,5 +68,5 @@ def test_oi2_singularity(self): OmicsIntegrator2.run(edges=EDGE_FILE, prizes=PRIZE_FILE, output_file=OUT_FILE, - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) assert OUT_FILE.exists() diff --git a/test/PathLinker/test_pathlinker.py b/test/PathLinker/test_pathlinker.py index ed9f10670..286389a69 100644 --- a/test/PathLinker/test_pathlinker.py +++ b/test/PathLinker/test_pathlinker.py @@ -3,6 +3,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.pathlinker import PathLinker @@ -60,5 +61,5 @@ def test_pathlinker_singularity(self): nodetypes=TEST_DIR+'input/sample-in-nodetypes.txt', network=TEST_DIR+'input/sample-in-net.txt', output_file=OUT_FILE_DEFAULT, - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) assert out_path.exists() diff --git a/test/RWR/test_RWR.py b/test/RWR/test_RWR.py index b0316ded0..5c01754d9 100644 --- a/test/RWR/test_RWR.py +++ b/test/RWR/test_RWR.py @@ -4,6 +4,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.rwr import RWR @@ -57,5 +58,5 @@ def test_rwr_singularity(self): nodes=Path(TEST_DIR, 'input','rwr-nodes.txt'), alpha=0.85, output_file=OUT_FILE, - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) assert OUT_FILE.exists() diff --git a/test/ResponseNet/test_rn.py b/test/ResponseNet/test_rn.py index 6b9fe05cf..9c8a585fc 100644 --- a/test/ResponseNet/test_rn.py +++ b/test/ResponseNet/test_rn.py @@ -4,6 +4,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.responsenet import ResponseNet @@ -56,7 +57,8 @@ def test_responsenet_singularity(self): ResponseNet.run(sources=TEST_DIR / 'input' / 'rn-sources.txt', targets=TEST_DIR / 'input' / 'rn-targets.txt', edges=TEST_DIR / 'input' / 'rn-edges.txt', - output_file=OUT_FILE) + output_file=OUT_FILE, + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) assert OUT_FILE.exists() assert filecmp.cmp(OUT_FILE, EXPECTED_FILE, shallow=True) diff --git a/test/ST_RWR/test_STRWR.py b/test/ST_RWR/test_STRWR.py index 898b24055..ccb88fd6c 100644 --- a/test/ST_RWR/test_STRWR.py +++ b/test/ST_RWR/test_STRWR.py @@ -4,6 +4,7 @@ import pytest +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.strwr import ST_RWR @@ -62,5 +63,5 @@ def test_strwr_singularity(self): targets=Path(TEST_DIR, 'input','strwr-targets.txt'), alpha=0.85, output_file=OUT_FILE, - container_framework="singularity") + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) assert OUT_FILE.exists() From 23d109b5f0db9ebe13a97546b7f7f468440c7d89 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Sat, 6 Sep 2025 07:31:52 +0000 Subject: [PATCH 03/13] fix(meo): specify correctly --- spras/meo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spras/meo.py b/spras/meo.py index 9461558ef..26a3a0c91 100644 --- a/spras/meo.py +++ b/spras/meo.py @@ -174,7 +174,7 @@ def run(edges=None, sources=None, targets=None, output_file=None, max_path_lengt properties_file_local = Path(out_dir, properties_file) write_properties(filename=properties_file_local, edges=edge_file, sources=source_file, targets=target_file, edge_output=mapped_output_file, path_output=mapped_path_output, - max_path_length=max_path_length, local_search=local_search, rand_restarts=rand_restarts, framework=container_framework) + max_path_length=max_path_length, local_search=local_search, rand_restarts=rand_restarts, framework=container_settings.framework) bind_path, properties_file = prepare_volume(str(properties_file_local), work_dir, container_settings) volumes.append(bind_path) @@ -182,7 +182,7 @@ def run(edges=None, sources=None, targets=None, output_file=None, max_path_lengt container_suffix = "meo" run_container_and_log('Maximum Edge Orientation', - container_framework, + container_suffix, command, volumes, work_dir, From 2ada9c40ec89ecf8b29f190f59e7efbf98ec0fb1 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Sat, 6 Sep 2025 07:41:29 +0000 Subject: [PATCH 04/13] fix: correct snakemake --- Snakefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Snakefile b/Snakefile index b65c7e13d..1c66e0baf 100644 --- a/Snakefile +++ b/Snakefile @@ -25,7 +25,7 @@ algorithm_params = _config.config.algorithm_params algorithm_directed = _config.config.algorithm_directed pca_params = _config.config.pca_params hac_params = _config.config.hac_params -FRAMEWORK = _config.config.container_settings.framework +container_settings = _config.config.container_settings include_aggregate_algo_eval = _config.config.analysis_include_evaluation_aggregate_algo # Return the dataset or gold_standard dictionary from the config file given the label @@ -271,7 +271,7 @@ rule reconstruct: # Remove the default placeholder parameter added for algorithms that have no parameters if 'spras_placeholder' in params: params.pop('spras_placeholder') - params['container_framework'] = FRAMEWORK + params['container_settings'] = container_settings runner.run(wildcards.algorithm, params) # Original pathway reconstruction output to universal output @@ -303,7 +303,7 @@ rule viz_cytoscape: output: session = SEP.join([out_dir, '{dataset}-cytoscape.cys']) run: - cytoscape.run_cytoscape(input.pathways, output.session, FRAMEWORK) + cytoscape.run_cytoscape(input.pathways, output.session, container_settings) # Write a single summary table for all pathways for each dataset From 4b5bbcb6bf114d5c2163f370aa0677a4357adb37 Mon Sep 17 00:00:00 2001 From: "Tristan F.-R." Date: Fri, 24 Oct 2025 22:47:29 +0000 Subject: [PATCH 05/13] style: fmt --- spras/containers.py | 2 +- spras/domino.py | 1 - test/AllPairs/test_ap.py | 2 +- test/BowTieBuilder/test_btb.py | 2 +- test/DOMINO/test_domino.py | 2 +- test/MEO/test_meo.py | 2 +- test/MinCostFlow/test_mcf.py | 2 +- test/OmicsIntegrator1/test_oi1.py | 2 +- test/OmicsIntegrator2/test_oi2.py | 2 +- test/PathLinker/test_pathlinker.py | 2 +- test/RWR/test_RWR.py | 2 +- test/ResponseNet/test_rn.py | 2 +- test/ST_RWR/test_STRWR.py | 2 +- 13 files changed, 12 insertions(+), 13 deletions(-) diff --git a/spras/containers.py b/spras/containers.py index e6f52cc03..60e9b73fe 100644 --- a/spras/containers.py +++ b/spras/containers.py @@ -561,4 +561,4 @@ def run_container_dsub(container: str, command: List[str], volumes: List[Tuple[P download_gcs(local_path=str(src), gcs_path=gcs_path, is_dir=True) # return location of dsub logs in WORKSPACE_BUCKET - return 'dsub logs: {logs}'.format(logs=flags['logging']) \ No newline at end of file + return 'dsub logs: {logs}'.format(logs=flags['logging']) diff --git a/spras/domino.py b/spras/domino.py index fd7aae97b..e060aa617 100644 --- a/spras/domino.py +++ b/spras/domino.py @@ -3,7 +3,6 @@ import pandas as pd - from spras.config.container_schema import ProcessedContainerSettings from spras.containers import ContainerError, prepare_volume, run_container_and_log from spras.interactome import ( diff --git a/test/AllPairs/test_ap.py b/test/AllPairs/test_ap.py index e4c921f37..f9920df04 100644 --- a/test/AllPairs/test_ap.py +++ b/test/AllPairs/test_ap.py @@ -4,9 +4,9 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.allpairs import AllPairs +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings # Note that we don't directly use the config in the test, but we need the config # to be initialized under the hood nonetheless. Initializing the config has implications diff --git a/test/BowTieBuilder/test_btb.py b/test/BowTieBuilder/test_btb.py index 6174e50bb..c6c1046ab 100644 --- a/test/BowTieBuilder/test_btb.py +++ b/test/BowTieBuilder/test_btb.py @@ -3,9 +3,9 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config from spras.btb import BowTieBuilder as BTB +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings config.init_from_file("config/config.yaml") diff --git a/test/DOMINO/test_domino.py b/test/DOMINO/test_domino.py index ed7469299..d53fc87cd 100644 --- a/test/DOMINO/test_domino.py +++ b/test/DOMINO/test_domino.py @@ -3,8 +3,8 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings from spras.domino import DOMINO, post_domino_id_transform, pre_domino_id_transform config.init_from_file("config/config.yaml") diff --git a/test/MEO/test_meo.py b/test/MEO/test_meo.py index 5bc564b39..cd5260f2e 100644 --- a/test/MEO/test_meo.py +++ b/test/MEO/test_meo.py @@ -3,8 +3,8 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings from spras.meo import MEO, write_properties config.init_from_file("config/config.yaml") diff --git a/test/MinCostFlow/test_mcf.py b/test/MinCostFlow/test_mcf.py index ee9f83e98..875743933 100644 --- a/test/MinCostFlow/test_mcf.py +++ b/test/MinCostFlow/test_mcf.py @@ -3,8 +3,8 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings from spras.mincostflow import MinCostFlow config.init_from_file("config/config.yaml") diff --git a/test/OmicsIntegrator1/test_oi1.py b/test/OmicsIntegrator1/test_oi1.py index 5e59314b9..458341c93 100644 --- a/test/OmicsIntegrator1/test_oi1.py +++ b/test/OmicsIntegrator1/test_oi1.py @@ -3,8 +3,8 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings from spras.omicsintegrator1 import OmicsIntegrator1, write_conf config.init_from_file("config/config.yaml") diff --git a/test/OmicsIntegrator2/test_oi2.py b/test/OmicsIntegrator2/test_oi2.py index e1fee011b..471a59882 100644 --- a/test/OmicsIntegrator2/test_oi2.py +++ b/test/OmicsIntegrator2/test_oi2.py @@ -3,8 +3,8 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings from spras.omicsintegrator2 import OmicsIntegrator2 config.init_from_file("config/config.yaml") diff --git a/test/PathLinker/test_pathlinker.py b/test/PathLinker/test_pathlinker.py index 286389a69..56f77ec7a 100644 --- a/test/PathLinker/test_pathlinker.py +++ b/test/PathLinker/test_pathlinker.py @@ -3,8 +3,8 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings from spras.pathlinker import PathLinker config.init_from_file("config/config.yaml") diff --git a/test/RWR/test_RWR.py b/test/RWR/test_RWR.py index 5c01754d9..08b42c369 100644 --- a/test/RWR/test_RWR.py +++ b/test/RWR/test_RWR.py @@ -4,8 +4,8 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings from spras.rwr import RWR config.init_from_file("config/config.yaml") diff --git a/test/ResponseNet/test_rn.py b/test/ResponseNet/test_rn.py index 9c8a585fc..2a3abccca 100644 --- a/test/ResponseNet/test_rn.py +++ b/test/ResponseNet/test_rn.py @@ -4,8 +4,8 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings from spras.responsenet import ResponseNet config.init_from_file("config/config.yaml") diff --git a/test/ST_RWR/test_STRWR.py b/test/ST_RWR/test_STRWR.py index ccb88fd6c..a4e5e1a37 100644 --- a/test/ST_RWR/test_STRWR.py +++ b/test/ST_RWR/test_STRWR.py @@ -4,8 +4,8 @@ import pytest -from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings import spras.config.config as config +from spras.config.container_schema import ContainerFramework, ProcessedContainerSettings from spras.strwr import ST_RWR config.init_from_file("config/config.yaml") From a47b3d81135d8d5282a1047d18b37aaad1a078dc Mon Sep 17 00:00:00 2001 From: "Tristan F.-R." Date: Fri, 24 Oct 2025 22:52:31 +0000 Subject: [PATCH 06/13] fix: drop re-introduced framework parameter bad merge conflict resolution --- spras/containers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spras/containers.py b/spras/containers.py index 60e9b73fe..6b1e28ebd 100644 --- a/spras/containers.py +++ b/spras/containers.py @@ -176,7 +176,7 @@ def env_to_items(environment: dict[str, str]) -> Iterator[str]: # TODO consider a better default environment variable # Follow docker-py's naming conventions (https://docker-py.readthedocs.io/en/stable/containers.html) # Technically the argument is an image, not a container, but we use container here. -def run_container(framework: str, container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, out_dir: str | os.PathLike, container_settings: ProcessedContainerSettings, environment: Optional[dict[str, str]] = None, network_disabled = False): +def run_container(container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, out_dir: str | os.PathLike, container_settings: ProcessedContainerSettings, environment: Optional[dict[str, str]] = None, network_disabled = False): """ Runs a command in the container using Singularity or Docker @param container_suffix: name of the DockerHub container without the 'docker://' prefix @@ -201,7 +201,7 @@ def run_container(framework: str, container_suffix: str, command: List[str], vol else: raise ValueError(f'{container_settings.framework} is not a recognized container framework. Choose "docker", "dsub", or "singularity".') -def run_container_and_log(name: str, framework: str, container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, out_dir: str | os.PathLike, container_settings: ProcessedContainerSettings, environment: Optional[dict[str, str]] = None, network_disabled=False): +def run_container_and_log(name: str, container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, out_dir: str | os.PathLike, container_settings: ProcessedContainerSettings, environment: Optional[dict[str, str]] = None, network_disabled=False): """ Runs a command in the container using Singularity or Docker with associated pretty printed messages. @param name: the display name of the running container for logging purposes @@ -219,7 +219,7 @@ def run_container_and_log(name: str, framework: str, container_suffix: str, comm print('Running {} on container framework "{}" on env {} with command: {}'.format(name, container_settings.framework, list(env_to_items(environment)), ' '.join(command)), flush=True) try: - out = run_container(framework=framework, container_suffix=container_suffix, command=command, volumes=volumes, working_dir=working_dir, out_dir=out_dir, container_settings=container_settings, environment=environment, network_disabled=network_disabled) + out = run_container(container_suffix=container_suffix, command=command, volumes=volumes, working_dir=working_dir, out_dir=out_dir, container_settings=container_settings, environment=environment, network_disabled=network_disabled) if out is not None: if isinstance(out, list): out = ''.join(out) From 5a2864ae3de684f5a41fff6197c7065ce8144899 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Sat, 25 Oct 2025 06:28:34 +0000 Subject: [PATCH 07/13] fix: move enable_profiling to container settings --- config/config.yaml | 22 +++++++++++----------- spras/config/config.py | 5 +---- spras/config/container_schema.py | 5 ++++- spras/config/schema.py | 1 - spras/containers.py | 4 ++-- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 7879f5e9f..11bac082a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -28,17 +28,17 @@ containers: # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs owner: reedcompbio -# Enabling profiling adds a file called 'usage-profile.tsv' to the output directory of each algorithm. -# The contents of this file describe the CPU utilization and peak memory consumption of the algorithm -# as seen by its runtime container. -# NOTE: Profiling is currently supported only when the container framework is set to apptainer/singularity -# and when the host system supports the 'cgroup' filesystem. -# When profiling via HTCondor, this assumes the current process is already in a two-level nested cgroup -# (introduced in HTCondor 24.8.0). To specify a minimum HTCondor version, use the following `requirements` -# expression: -# -# requirements = versionGE(split(Target.CondorVersion)[1], "24.8.0") && (isenforcingdiskusage =!= true) -enable_profiling: false + # Enabling profiling adds a file called 'usage-profile.tsv' to the output directory of each algorithm. + # The contents of this file describe the CPU utilization and peak memory consumption of the algorithm + # as seen by its runtime container. + # NOTE: Profiling is currently supported only when the container framework is set to apptainer/singularity + # and when the host system supports the 'cgroup' filesystem. + # When profiling via HTCondor, this assumes the current process is already in a two-level nested cgroup + # (introduced in HTCondor 24.8.0). To specify a minimum HTCondor version, use the following `requirements` + # expression: + # + # requirements = versionGE(split(Target.CondorVersion)[1], "24.8.0") && (isenforcingdiskusage =!= true) + enable_profiling: false # This list of algorithms should be generated by a script which checks the filesystem for installs. # It shouldn't be changed by mere mortals. (alternatively, we could add a path to executable for each algorithm diff --git a/spras/config/config.py b/spras/config/config.py index 25e6f72de..cb19b2b1d 100644 --- a/spras/config/config.py +++ b/spras/config/config.py @@ -66,8 +66,6 @@ def __init__(self, raw_config: dict[str, Any]): # Directory used for storing output self.out_dir = parsed_raw_config.reconstruction_settings.locations.reconstruction_dir - # A Boolean indicating whether to enable container runtime profiling (apptainer/singularity only) - self.enable_profiling = False # A dictionary to store configured datasets against which SPRAS will be run self.datasets = None # A dictionary to store configured gold standard data against output of SPRAS runs @@ -292,9 +290,8 @@ def process_config(self, raw_config: RawConfig): # Set up a few top-level config variables self.out_dir = raw_config.reconstruction_settings.locations.reconstruction_dir - if raw_config.enable_profiling and raw_config.containers.framework not in ["singularity", "apptainer"]: + if raw_config.containers.enable_profiling and raw_config.containers.framework not in ["singularity", "apptainer"]: warnings.warn("enable_profiling is set to true, but the container framework is not singularity/apptainer. This setting will have no effect.", stacklevel=2) - self.enable_profiling = raw_config.enable_profiling self.process_datasets(raw_config) self.process_algorithms(raw_config) diff --git a/spras/config/container_schema.py b/spras/config/container_schema.py index e85d22e60..2167ae9e4 100644 --- a/spras/config/container_schema.py +++ b/spras/config/container_schema.py @@ -33,15 +33,18 @@ class ContainerRegistry(BaseModel): class ContainerSettings(BaseModel): framework: ContainerFramework = ContainerFramework.docker unpack_singularity: bool = False + enable_profiling: bool = False + "A Boolean indicating whether to enable container runtime profiling (apptainer/singularity only)" registry: ContainerRegistry - model_config = ConfigDict(extra='forbid') + model_config = ConfigDict(extra='forbid', use_attribute_docstrings=True) @dataclass class ProcessedContainerSettings: framework: ContainerFramework = ContainerFramework.docker unpack_singularity: bool = False prefix: str = DEFAULT_CONTAINER_PREFIX + enable_profiling: bool = False hash_length: int = 7 """ The hash length for container-specific usage. This does not appear in diff --git a/spras/config/schema.py b/spras/config/schema.py index a1936b0c0..8aa067a53 100644 --- a/spras/config/schema.py +++ b/spras/config/schema.py @@ -139,7 +139,6 @@ class ReconstructionSettings(BaseModel): class RawConfig(BaseModel): containers: ContainerSettings - enable_profiling: bool = False hash_length: int = DEFAULT_HASH_LENGTH "The length of the hash used to identify a parameter combination" diff --git a/spras/containers.py b/spras/containers.py index 6b1e28ebd..f9c85a927 100644 --- a/spras/containers.py +++ b/spras/containers.py @@ -386,7 +386,7 @@ def run_container_singularity(container: str, command: List[str], volumes: List[ # Handle unpacking singularity image if needed. Potentially needed for running nested unprivileged containers expanded_image = None - if config.config.container_settings.unpack_singularity: + if config.unpack_singularity: # The incoming image string is of the format //: e.g. # hub.docker.com/reedcompbio/spras:latest # Here we first produce a .sif image using the image name and tag (base_cont) @@ -417,7 +417,7 @@ def run_container_singularity(container: str, command: List[str], volumes: List[ # If not using the expanded sandbox image, we still need to prepend the docker:// prefix # so apptainer knows to pull and convert the image format from docker to apptainer. image_to_run = expanded_image if expanded_image else "docker://" + container - if config.config.enable_profiling: + if config.enable_profiling: # We won't end up using the spython client if profiling is enabled because # we need to run everything manually to set up the cgroup # Build the apptainer run command, which gets passed to the cgroup wrapper script From a7e56a1434a14cf1eb8e95c51713f097d3ddd536 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Fri, 24 Oct 2025 23:35:38 -0700 Subject: [PATCH 08/13] fix(meo): reorder run_container_and_log params --- spras/meo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spras/meo.py b/spras/meo.py index 42776c264..0f1545282 100644 --- a/spras/meo.py +++ b/spras/meo.py @@ -186,8 +186,8 @@ def run(edges=None, sources=None, targets=None, output_file=None, max_path_lengt command, volumes, work_dir, - container_settings, - out_dir) + out_dir, + container_settings) properties_file_local.unlink(missing_ok=True) From cb7cbc8c7cf1221b916c4f6eb7aa918527b1a1a2 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Fri, 24 Oct 2025 23:49:31 -0700 Subject: [PATCH 09/13] fix: correct out_dir for cytoscape --- spras/analysis/cytoscape.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spras/analysis/cytoscape.py b/spras/analysis/cytoscape.py index c7a69bf37..ea0d254fd 100644 --- a/spras/analysis/cytoscape.py +++ b/spras/analysis/cytoscape.py @@ -48,12 +48,15 @@ def run_cytoscape(pathways: List[Union[str, PurePath]], output_file: str, contai # Provided the mapped pathway file path and the original file path as the label Cytoscape command.extend(['--pathway', f'{mapped_pathway}|{pathway}']) + out_dir = Path(output_file).parent + container_suffix = "py4cytoscape:v3" run_container_and_log('Cytoscape', container_suffix, command, volumes, work_dir, + out_dir, container_settings, env) rmtree(cytoscape_output_dir) From ae00175db20d5c76235d65c05de6fd8391955f29 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Sun, 26 Oct 2025 03:19:57 +0000 Subject: [PATCH 10/13] test(apsp): indicate singularity unpacking via container settings --- test/AllPairs/test_ap.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/AllPairs/test_ap.py b/test/AllPairs/test_ap.py index f9920df04..460761e44 100644 --- a/test/AllPairs/test_ap.py +++ b/test/AllPairs/test_ap.py @@ -81,15 +81,12 @@ def test_allpairs_singularity(self): def test_allpairs_singularity_unpacked(self): out_path = OUT_DIR / 'sample-out-unpack.txt' out_path.unlink(missing_ok=True) - # Indicate via config mechanism that we want to unpack the Singularity container - config.config.container_settings.unpack_singularity = True AllPairs.run( nodetypes=str(TEST_DIR / 'input/sample-in-nodetypes.txt'), network=str(TEST_DIR / 'input/sample-in-net.txt'), directed_flag=str(TEST_DIR / 'input' / 'directed-flag-false.txt'), output_file=str(out_path), - container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity)) - config.config.container_settings.unpack_singularity = False + container_settings=ProcessedContainerSettings(framework=ContainerFramework.singularity, unpack_singularity=True)) assert out_path.exists() def test_allpairs_correctness(self): From 8c96939732aa54aeedaf2d045d2d270de570336d Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Sat, 1 Nov 2025 08:14:33 +0000 Subject: [PATCH 11/13] fix: apply two suggestions --- spras/containers.py | 2 +- test/test_util.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/spras/containers.py b/spras/containers.py index f9c85a927..124b97411 100644 --- a/spras/containers.py +++ b/spras/containers.py @@ -199,7 +199,7 @@ def run_container(container_suffix: str, command: List[str], volumes: List[Tuple elif normalized_framework == 'dsub': return run_container_dsub(container, command, volumes, working_dir, environment) else: - raise ValueError(f'{container_settings.framework} is not a recognized container framework. Choose "docker", "dsub", or "singularity".') + raise ValueError(f'{container_settings.framework} is not a recognized container framework. Choose "docker", "dsub", "apptainer", or "singularity".') def run_container_and_log(name: str, container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, out_dir: str | os.PathLike, container_settings: ProcessedContainerSettings, environment: Optional[dict[str, str]] = None, network_disabled=False): """ diff --git a/test/test_util.py b/test/test_util.py index c18a35f75..8ee8c8df1 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -3,6 +3,7 @@ import pytest import spras.config.config as config +from spras.config.container_schema import ProcessedContainerSettings from spras.containers import convert_docker_path, prepare_path_docker, prepare_volume from spras.util import hash_params_sha1_base32 @@ -41,7 +42,7 @@ def test_hash_params_sha1_base32(self): ('test/OmicsIntegrator1/output', PurePosixPath('/spras'), '/spras/TNDO5TR/output'), ('../src', '/spras', '/spras/NNBVZ6X/src')]) def test_prepare_volume(self, filename, volume_base, expected_filename): - _, container_filename = prepare_volume(filename, volume_base, config.config.container_settings) + _, container_filename = prepare_volume(filename, volume_base, ProcessedContainerSettings()) assert container_filename == expected_filename def test_convert_docker_path(self): From 7977db115b845f19462d3f8a44d6d79e09e084a1 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Sat, 1 Nov 2025 08:17:44 +0000 Subject: [PATCH 12/13] docs(cytoscape): mention out_dir usage for profiling --- spras/analysis/cytoscape.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spras/analysis/cytoscape.py b/spras/analysis/cytoscape.py index ea0d254fd..b1e0455ea 100644 --- a/spras/analysis/cytoscape.py +++ b/spras/analysis/cytoscape.py @@ -48,6 +48,8 @@ def run_cytoscape(pathways: List[Union[str, PurePath]], output_file: str, contai # Provided the mapped pathway file path and the original file path as the label Cytoscape command.extend(['--pathway', f'{mapped_pathway}|{pathway}']) + # out_dir does not contain all Cytoscape output files, but it is where + # profiling's `usage-profile.tsv` file goes if `enable_profiling` is enabled. out_dir = Path(output_file).parent container_suffix = "py4cytoscape:v3" From 50d7b50eb8dc88c249157adfe1abe1b96a6ebc57 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Mon, 3 Nov 2025 00:09:23 +0000 Subject: [PATCH 13/13] revert cytoscape out_dir --- spras/analysis/cytoscape.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/spras/analysis/cytoscape.py b/spras/analysis/cytoscape.py index b1e0455ea..e84899506 100644 --- a/spras/analysis/cytoscape.py +++ b/spras/analysis/cytoscape.py @@ -48,17 +48,15 @@ def run_cytoscape(pathways: List[Union[str, PurePath]], output_file: str, contai # Provided the mapped pathway file path and the original file path as the label Cytoscape command.extend(['--pathway', f'{mapped_pathway}|{pathway}']) - # out_dir does not contain all Cytoscape output files, but it is where - # profiling's `usage-profile.tsv` file goes if `enable_profiling` is enabled. - out_dir = Path(output_file).parent - container_suffix = "py4cytoscape:v3" run_container_and_log('Cytoscape', container_suffix, command, volumes, work_dir, - out_dir, + # TODO: doesn't work on Singularity + # (https://github.com/Reed-CompBio/spras/pull/390/files#r2485100875) + None, container_settings, env) rmtree(cytoscape_output_dir)