diff --git a/Snakefile b/Snakefile index 1cc6ffd3c..02f019e8d 100644 --- a/Snakefile +++ b/Snakefile @@ -25,7 +25,7 @@ algorithm_params = _config.config.algorithm_params algorithm_directed = _config.config.algorithm_directed pca_params = _config.config.pca_params hac_params = _config.config.hac_params -FRAMEWORK = _config.config.container_framework +FRAMEWORK = _config.config.container_settings.framework include_aggregate_algo_eval = _config.config.analysis_include_evaluation_aggregate_algo # Return the dataset or gold_standard dictionary from the config file given the label diff --git a/config/config.yaml b/config/config.yaml index 40f180b31..7879f5e9f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -3,11 +3,30 @@ # The length of the hash used to identify a parameter combination hash_length: 7 -# Specify the container framework used by each PRM wrapper. Valid options include: -# - docker (default if not specified) -# - singularity OR apptainer -- Apptainer (formerly Singularity) is useful in HPC/HTC environments where docker isn't allowed -# - dsub -- experimental with limited support, used for running on Google Cloud -container_framework: docker +containers: + # Specify the container framework used by each PRM wrapper. Valid options include: + # - docker (default if not specified) + # - singularity OR apptainer -- Apptainer (formerly Singularity) is useful in HPC/HTC environments where docker isn't allowed + # - dsub -- experimental with limited support, used for running on Google Cloud + framework: docker + + # Only used if framework is set to singularity/apptainer, this will unpack the containers + # to the local filesystem. This is useful when PRM containers need to run inside another container, + # such as would be the case in an HTCondor/OSPool environment. + # NOTE: This unpacks containers to the local filesystem, which will take up space in a way + # that persists after the workflow is complete. To clean up the unpacked containers, the user must + # manually delete them. For convenience, these unpacked files will exist in the current working directory + # under `unpacked`. + unpack_singularity: false + + # Allow the user to configure which container registry containers should be pulled from + # Note that this assumes container names are consistent across registries, and that the + # registry being passed doesn't require authentication for pull actions + registry: + base_url: docker.io + # The owner or project of the registry + # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs + owner: reedcompbio # Enabling profiling adds a file called 'usage-profile.tsv' to the output directory of each algorithm. # The contents of this file describe the CPU utilization and peak memory consumption of the algorithm @@ -21,24 +40,6 @@ container_framework: docker # requirements = versionGE(split(Target.CondorVersion)[1], "24.8.0") && (isenforcingdiskusage =!= true) enable_profiling: false -# Only used if container_framework is set to singularity/apptainer, this will unpack the containers -# to the local filesystem. This is useful when PRM containers need to run inside another container, -# such as would be the case in an HTCondor/OSPool environment. -# NOTE: This unpacks containers to the local filesystem, which will take up space in a way -# that persists after the workflow is complete. To clean up the unpacked containers, the user must -# manually delete them. For convenience, these unpacked files will exist in the current working directory -# under `unpacked`. -unpack_singularity: false - -# Allow the user to configure which container registry containers should be pulled from -# Note that this assumes container names are consistent across registries, and that the -# registry being passed doesn't require authentication for pull actions -container_registry: - base_url: docker.io - # The owner or project of the registry - # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs - owner: reedcompbio - # This list of algorithms should be generated by a script which checks the filesystem for installs. # It shouldn't be changed by mere mortals. (alternatively, we could add a path to executable for each algorithm # in the list to reduce the number of assumptions of the program at the cost of making the config a little more involved) diff --git a/config/egfr.yaml b/config/egfr.yaml index 667cb55f0..25e56ab25 100644 --- a/config/egfr.yaml +++ b/config/egfr.yaml @@ -1,28 +1,31 @@ # The length of the hash used to identify a parameter combination hash_length: 7 -# Specify the container framework used by each PRM wrapper. Valid options include: -# - docker (default if not specified) -# - singularity -- Also known as apptainer, useful in HPC/HTC environments where docker isn't allowed -# - dsub -- experimental with limited support, used for running on Google Cloud -container_framework: docker +containers: + # Specify the container framework used by each PRM wrapper. Valid options include: + # - docker (default if not specified) + # - singularity -- Also known as apptainer, useful in HPC/HTC environments where docker isn't allowed + # - dsub -- experimental with limited support, used for running on Google Cloud with the All of Us cloud environment. + # - There is no support for other environments at the moment. + framework: docker -# Only used if container_framework is set to singularity, this will unpack the singularity containers -# to the local filesystem. This is useful when PRM containers need to run inside another container, -# such as would be the case in an HTCondor/OSPool environment. -# NOTE: This unpacks singularity containers to the local filesystem, which will take up space in a way -# that persists after the workflow is complete. To clean up the unpacked containers, the user must -# manually delete them. -unpack_singularity: false + # Only used if framework is set to singularity, this will unpack the singularity containers + # to the local filesystem. This is useful when PRM containers need to run inside another container, + # such as would be the case in an HTCondor/OSPool environment. + # NOTE: This unpacks singularity containers to the local filesystem, which will take up space in a way + # that persists after the workflow is complete. To clean up the unpacked containers, the user must + # manually delete them. For convenience, these unpacked files will exist in the current working directory + # under `unpacked`. + unpack_singularity: false -# Allow the user to configure which container registry containers should be pulled from -# Note that this assumes container names are consistent across registries, and that the -# registry being passed doesn't require authentication for pull actions -container_registry: - base_url: docker.io - # The owner or project of the registry - # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs - owner: reedcompbio + # Allow the user to configure which container registry containers should be pulled from + # Note that this assumes container names are consistent across registries, and that the + # registry being passed doesn't require authentication for pull actions + registry: + base_url: docker.io + # The owner or project of the registry + # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs + owner: reedcompbio algorithms: - name: pathlinker diff --git a/docker-wrappers/SPRAS/example_config.yaml b/docker-wrappers/SPRAS/example_config.yaml index 87e996a9c..db1c2dbbf 100644 --- a/docker-wrappers/SPRAS/example_config.yaml +++ b/docker-wrappers/SPRAS/example_config.yaml @@ -3,21 +3,31 @@ # The length of the hash used to identify a parameter combination hash_length: 7 -# Specify the container framework. Current supported versions include 'docker' and -# 'singularity'. If container_framework is not specified, SPRAS will default to docker. -container_framework: singularity - -# Unpack singularity. See config/config.yaml for details. -unpack_singularity: true - -# Allow the user to configure which container registry containers should be pulled from -# Note that this assumes container names are consistent across registries, and that the -# registry being passed doesn't require authentication for pull actions -container_registry: - base_url: docker.io - # The owner or project of the registry - # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs - owner: reedcompbio +containers: + # Specify the container framework used by each PRM wrapper. Valid options include: + # - docker (default if not specified) + # - singularity OR apptainer -- Apptainer (formerly Singularity) is useful in HPC/HTC environments where docker isn't allowed + # - dsub -- experimental with limited support, used for running on Google Cloud + framework: singularity + + # Only used if framework is set to singularity/apptainer, this will unpack the containers + # to the local filesystem. This is useful when PRM containers need to run inside another container, + # such as would be the case in an HTCondor/OSPool environment. + # NOTE: This unpacks containers to the local filesystem, which will take up space in a way + # that persists after the workflow is complete. To clean up the unpacked containers, the user must + # manually delete them. For convenience, these unpacked files will exist in the current working directory + # under `unpacked`. + # Here, we unpack it since we're running on HTCondor. + unpack_singularity: true + + # Allow the user to configure which container registry containers should be pulled from + # Note that this assumes container names are consistent across registries, and that the + # registry being passed doesn't require authentication for pull actions + registry: + base_url: docker.io + # The owner or project of the registry + # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs + owner: reedcompbio # This list of algorithms should be generated by a script which checks the filesystem for installs. # It shouldn't be changed by mere mortals. (alternatively, we could add a path to executable for each algorithm diff --git a/docs/_static/config/beginner.yaml b/docs/_static/config/beginner.yaml index 951dd2834..1ddda6dc2 100644 --- a/docs/_static/config/beginner.yaml +++ b/docs/_static/config/beginner.yaml @@ -1,9 +1,10 @@ hash_length: 7 -container_framework: docker -unpack_singularity: false -container_registry: - base_url: docker.io - owner: reedcompbio +containers: + framework: docker + unpack_singularity: false + registry: + base_url: docker.io + owner: reedcompbio # Each algorithm has an 'include' parameter. By toggling 'include' to true/false the user can change # which algorithms are run in a given experiment. diff --git a/docs/_static/config/intermediate.yaml b/docs/_static/config/intermediate.yaml index 78f5a7489..1f0ba2eb5 100644 --- a/docs/_static/config/intermediate.yaml +++ b/docs/_static/config/intermediate.yaml @@ -1,9 +1,10 @@ hash_length: 7 -container_framework: docker -unpack_singularity: false -container_registry: - base_url: docker.io - owner: reedcompbio +containers: + framework: docker + unpack_singularity: false + registry: + base_url: docker.io + owner: reedcompbio # Each algorithm has an 'include' parameter. By toggling 'include' to true/false the user can change # which algorithms are run in a given experiment. diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index be5f656f1..0d8a542ff 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -285,7 +285,7 @@ Local Neighborhood has no other parameters. Optionally set ``include: false`` for the other pathway reconstruction algorithms to make testing faster. -The config file has an option ``owner`` under the ``container_registry`` +The config file has an option ``owner`` under the ``containers.registry`` settings that controls which Docker Hub account will be used when pulling Docker images. The same Docker Hub account will be used for all images and cannot currently be set different for each algorithm. Set the diff --git a/docs/htcondor.rst b/docs/htcondor.rst index 6103e1414..4a96d4487 100644 --- a/docs/htcondor.rst +++ b/docs/htcondor.rst @@ -71,7 +71,7 @@ it uses the SPRAS apptainer image you created: container_image = < your spras image >.sif Make sure to modify the configuration file to have -``unpack_singularity`` set to ``true``, and ``container_framework`` set +``unpack_singularity`` set to ``true``, and ``containers.framework`` set to ``singularity``: else, the workflow will (likely) fail. Then run ``condor_submit spras.sub``, which will submit SPRAS to diff --git a/docs/tutorial/advanced.rst b/docs/tutorial/advanced.rst index 7a20cc68f..8f7e8b645 100644 --- a/docs/tutorial/advanced.rst +++ b/docs/tutorial/advanced.rst @@ -177,6 +177,7 @@ The global workflow control section in the configuration file allows a user to s .. code-block:: yaml - container_framework: docker + containers: + framework: docker The frameworks include Docker, Apptainer/Singularity, or dsub diff --git a/spras/config/config.py b/spras/config/config.py index d032e126e..25e6f72de 100644 --- a/spras/config/config.py +++ b/spras/config/config.py @@ -7,7 +7,7 @@ value. For example import spras.config.config as config -container_framework = config.config.container_framework +container_framework = config.config.container_settings.framework will grab the top level registry configuration option as it appears in the config file """ @@ -22,7 +22,8 @@ import numpy as np import yaml -from spras.config.schema import ContainerFramework, RawConfig +from spras.config.container_schema import ProcessedContainerSettings +from spras.config.schema import RawConfig from spras.util import NpHashEncoder, hash_params_sha1_base32 config = None @@ -65,12 +66,6 @@ def __init__(self, raw_config: dict[str, Any]): # Directory used for storing output self.out_dir = parsed_raw_config.reconstruction_settings.locations.reconstruction_dir - # Container framework used by PRMs. Valid options are "docker", "dsub", and "singularity" - self.container_framework = None - # The container prefix (host and organization) to use for images. Default is "docker.io/reedcompbio" - self.container_prefix: str = DEFAULT_CONTAINER_PREFIX - # A Boolean specifying whether to unpack singularity containers. Default is False - self.unpack_singularity = False # A Boolean indicating whether to enable container runtime profiling (apptainer/singularity only) self.enable_profiling = False # A dictionary to store configured datasets against which SPRAS will be run @@ -79,6 +74,8 @@ def __init__(self, raw_config: dict[str, Any]): self.gold_standards = None # The hash length SPRAS will use to identify parameter combinations. self.hash_length = parsed_raw_config.hash_length + # Container settings used by PRMs. + self.container_settings = ProcessedContainerSettings.from_container_settings(parsed_raw_config.containers, self.hash_length) # The list of algorithms to run in the workflow. Each is a dict with 'name' as an expected key. self.algorithms = None # A nested dict mapping algorithm names to dicts that map parameter hashes to parameter combinations. @@ -295,20 +292,7 @@ def process_config(self, raw_config: RawConfig): # Set up a few top-level config variables self.out_dir = raw_config.reconstruction_settings.locations.reconstruction_dir - if raw_config.container_framework == ContainerFramework.dsub: - warnings.warn("'dsub' framework integration is experimental and may not be fully supported.", stacklevel=2) - self.container_framework = raw_config.container_framework - - # Unpack settings for running in singularity mode. Needed when running PRM containers if already in a container. - if raw_config.unpack_singularity and self.container_framework != "singularity": - warnings.warn("unpack_singularity is set to True, but the container framework is not singularity. This setting will have no effect.", stacklevel=2) - self.unpack_singularity = raw_config.unpack_singularity - - # Grab registry from the config, and if none is provided default to docker - if raw_config.container_registry and raw_config.container_registry.base_url != "" and raw_config.container_registry.owner != "": - self.container_prefix = raw_config.container_registry.base_url + "/" + raw_config.container_registry.owner - - if raw_config.enable_profiling and raw_config.container_framework not in ["singularity", "apptainer"]: + if raw_config.enable_profiling and raw_config.containers.framework not in ["singularity", "apptainer"]: warnings.warn("enable_profiling is set to true, but the container framework is not singularity/apptainer. This setting will have no effect.", stacklevel=2) self.enable_profiling = raw_config.enable_profiling diff --git a/spras/config/container_schema.py b/spras/config/container_schema.py new file mode 100644 index 000000000..e85d22e60 --- /dev/null +++ b/spras/config/container_schema.py @@ -0,0 +1,77 @@ +""" +The separate container schema specification file. +For information about pydantic, see schema.py. + +We move this to a separate file to allow `containers.py` to explicitly take in +this subsection of the configuration. +""" + +import warnings +from dataclasses import dataclass + +from pydantic import BaseModel, ConfigDict + +from spras.config.util import CaseInsensitiveEnum + +DEFAULT_CONTAINER_PREFIX = "docker.io/reedcompbio" + +class ContainerFramework(CaseInsensitiveEnum): + docker = 'docker' + singularity = 'singularity' + apptainer = 'apptainer' + dsub = 'dsub' + +class ContainerRegistry(BaseModel): + base_url: str = "docker.io" + "The domain of the registry" + + owner: str = "reedcompbio" + "The owner or project of the registry" + + model_config = ConfigDict(extra='forbid', use_attribute_docstrings=True) + +class ContainerSettings(BaseModel): + framework: ContainerFramework = ContainerFramework.docker + unpack_singularity: bool = False + registry: ContainerRegistry + + model_config = ConfigDict(extra='forbid') + +@dataclass +class ProcessedContainerSettings: + framework: ContainerFramework = ContainerFramework.docker + unpack_singularity: bool = False + prefix: str = DEFAULT_CONTAINER_PREFIX + hash_length: int = 7 + """ + The hash length for container-specific usage. This does not appear in + the output folder, but it may show up in logs, and usually never needs + to be tinkered with. This will be the top-level `hash_length` specified + in the config. + + We prefer this `hash_length` in our container-running logic to + avoid a (future) dependency diamond. + """ + + @staticmethod + def from_container_settings(settings: ContainerSettings, hash_length: int) -> "ProcessedContainerSettings": + if settings.framework == ContainerFramework.dsub: + warnings.warn("'dsub' framework integration is experimental and may not be fully supported.", stacklevel=2) + container_framework = settings.framework + + # Unpack settings for running in singularity mode. Needed when running PRM containers if already in a container. + if settings.unpack_singularity and container_framework != "singularity": + warnings.warn("unpack_singularity is set to True, but the container framework is not singularity. This setting will have no effect.", stacklevel=2) + unpack_singularity = settings.unpack_singularity + + # Grab registry from the config, and if none is provided default to docker + container_prefix = DEFAULT_CONTAINER_PREFIX + if settings.registry and settings.registry.base_url != "" and settings.registry.owner != "": + container_prefix = settings.registry.base_url + "/" + settings.registry.owner + + return ProcessedContainerSettings( + framework=container_framework, + unpack_singularity=unpack_singularity, + prefix=container_prefix, + hash_length=hash_length + ) diff --git a/spras/config/schema.py b/spras/config/schema.py index 2b46aaf1c..a1936b0c0 100644 --- a/spras/config/schema.py +++ b/spras/config/schema.py @@ -13,8 +13,9 @@ import re from typing import Annotated, Optional -from pydantic import AfterValidator, BaseModel, ConfigDict, Field +from pydantic import AfterValidator, BaseModel, ConfigDict +from spras.config.container_schema import ContainerSettings from spras.config.util import CaseInsensitiveEnum # Most options here have an `include` property, @@ -89,18 +90,6 @@ def validate(label: str): return label return validate -class ContainerFramework(CaseInsensitiveEnum): - docker = 'docker' - singularity = 'singularity' - apptainer = 'apptainer' - dsub = 'dsub' - -class ContainerRegistry(BaseModel): - base_url: str - owner: str = Field(description="The owner or project of the registry") - - model_config = ConfigDict(extra='forbid') - class AlgorithmParams(BaseModel): include: bool directed: Optional[bool] = None @@ -149,10 +138,7 @@ class ReconstructionSettings(BaseModel): model_config = ConfigDict(extra='forbid') class RawConfig(BaseModel): - # TODO: move these container values to a nested container key - container_framework: ContainerFramework = ContainerFramework.docker - unpack_singularity: bool = False - container_registry: ContainerRegistry + containers: ContainerSettings enable_profiling: bool = False hash_length: int = DEFAULT_HASH_LENGTH diff --git a/spras/containers.py b/spras/containers.py index 4706d7257..9c8c4b3f0 100644 --- a/spras/containers.py +++ b/spras/containers.py @@ -191,7 +191,7 @@ def run_container(framework: str, container_suffix: str, command: List[str], vol """ normalized_framework = framework.casefold() - container = config.config.container_prefix + "/" + container_suffix + container = config.config.container_settings.prefix + "/" + container_suffix if normalized_framework == 'docker': return run_container_docker(container, command, volumes, working_dir, environment, network_disabled) elif normalized_framework == 'singularity' or normalized_framework == "apptainer": @@ -385,7 +385,7 @@ def run_container_singularity(container: str, command: List[str], volumes: List[ # Handle unpacking singularity image if needed. Potentially needed for running nested unprivileged containers expanded_image = None - if config.config.unpack_singularity: + if config.config.container_settings.unpack_singularity: # The incoming image string is of the format //: e.g. # hub.docker.com/reedcompbio/spras:latest # Here we first produce a .sif image using the image name and tag (base_cont) @@ -471,7 +471,7 @@ def prepare_volume(filename: Union[str, PurePath], volume_base: Union[str, PureP if isinstance(filename, PurePath): filename = str(filename) - filename_hash = hash_filename(filename, config.config.hash_length) + filename_hash = hash_filename(filename, config.config.container_settings.hash_length) dest = PurePosixPath(base_path, filename_hash) abs_filename = Path(filename).resolve() diff --git a/test/AllPairs/test_ap.py b/test/AllPairs/test_ap.py index a8291f72f..355033f9f 100644 --- a/test/AllPairs/test_ap.py +++ b/test/AllPairs/test_ap.py @@ -81,14 +81,14 @@ def test_allpairs_singularity_unpacked(self): out_path = OUT_DIR / 'sample-out-unpack.txt' out_path.unlink(missing_ok=True) # Indicate via config mechanism that we want to unpack the Singularity container - config.config.unpack_singularity = True + config.config.container_settings.unpack_singularity = True AllPairs.run( nodetypes=str(TEST_DIR / 'input/sample-in-nodetypes.txt'), network=str(TEST_DIR / 'input/sample-in-net.txt'), directed_flag=str(TEST_DIR / 'input' / 'directed-flag-false.txt'), output_file=str(out_path), container_framework="singularity") - config.config.unpack_singularity = False + config.config.container_settings.unpack_singularity = False assert out_path.exists() def test_allpairs_correctness(self): diff --git a/test/analysis/input/config.yaml b/test/analysis/input/config.yaml index abde6f979..15a5572fa 100644 --- a/test/analysis/input/config.yaml +++ b/test/analysis/input/config.yaml @@ -1,26 +1,31 @@ # The length of the hash used to identify a parameter combination hash_length: 7 -# Specify the container framework. Current supported versions include 'docker' and -# 'singularity'. If container_framework is not specified, SPRAS will default to docker. -container_framework: docker +containers: + # Specify the container framework used by each PRM wrapper. Valid options include: + # - docker (default if not specified) + # - singularity -- Also known as apptainer, useful in HPC/HTC environments where docker isn't allowed + # - dsub -- experimental with limited support, used for running on Google Cloud with the All of Us cloud environment. + # - There is no support for other environments at the moment. + framework: docker -# Only used if container_framework is set to singularity, this will unpack the singularity containers -# to the local filesystem. This is useful when PRM containers need to run inside another container, -# such as would be the case in an HTCondor/OSPool environment. -# NOTE: This unpacks singularity containers to the local filesystem, which will take up space in a way -# that persists after the workflow is complete. To clean up the unpacked containers, the user must -# manually delete them. -unpack_singularity: false + # Only used if container_framework is set to singularity, this will unpack the singularity containers + # to the local filesystem. This is useful when PRM containers need to run inside another container, + # such as would be the case in an HTCondor/OSPool environment. + # NOTE: This unpacks singularity containers to the local filesystem, which will take up space in a way + # that persists after the workflow is complete. To clean up the unpacked containers, the user must + # manually delete them. For convenience, these unpacked files will exist in the current working directory + # under `unpacked`. + unpack_singularity: false -# Allow the user to configure which container registry containers should be pulled from -# Note that this assumes container names are consistent across registries, and that the -# registry being passed doesn't require authentication for pull actions -container_registry: - base_url: docker.io - # The owner or project of the registry - # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs - owner: reedcompbio + # Allow the user to configure which container registry containers should be pulled from + # Note that this assumes container names are consistent across registries, and that the + # registry being passed doesn't require authentication for pull actions + registry: + base_url: docker.io + # The owner or project of the registry + # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs + owner: reedcompbio algorithms: - name: "pathlinker" diff --git a/test/analysis/input/egfr.yaml b/test/analysis/input/egfr.yaml index da4560df9..d26bded2d 100644 --- a/test/analysis/input/egfr.yaml +++ b/test/analysis/input/egfr.yaml @@ -1,26 +1,31 @@ # The length of the hash used to identify a parameter combination hash_length: 7 -# Specify the container framework. Current supported versions include 'docker' and -# 'singularity'. If container_framework is not specified, SPRAS will default to docker. -container_framework: docker +containers: + # Specify the container framework used by each PRM wrapper. Valid options include: + # - docker (default if not specified) + # - singularity -- Also known as apptainer, useful in HPC/HTC environments where docker isn't allowed + # - dsub -- experimental with limited support, used for running on Google Cloud with the All of Us cloud environment. + # - There is no support for other environments at the moment. + framework: docker -# Only used if container_framework is set to singularity, this will unpack the singularity containers -# to the local filesystem. This is useful when PRM containers need to run inside another container, -# such as would be the case in an HTCondor/OSPool environment. -# NOTE: This unpacks singularity containers to the local filesystem, which will take up space in a way -# that persists after the workflow is complete. To clean up the unpacked containers, the user must -# manually delete them. -unpack_singularity: false + # Only used if container_framework is set to singularity, this will unpack the singularity containers + # to the local filesystem. This is useful when PRM containers need to run inside another container, + # such as would be the case in an HTCondor/OSPool environment. + # NOTE: This unpacks singularity containers to the local filesystem, which will take up space in a way + # that persists after the workflow is complete. To clean up the unpacked containers, the user must + # manually delete them. For convenience, these unpacked files will exist in the current working directory + # under `unpacked`. + unpack_singularity: false -# Allow the user to configure which container registry containers should be pulled from -# Note that this assumes container names are consistent across registries, and that the -# registry being passed doesn't require authentication for pull actions -container_registry: - base_url: docker.io - # The owner or project of the registry - # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs - owner: reedcompbio + # Allow the user to configure which container registry containers should be pulled from + # Note that this assumes container names are consistent across registries, and that the + # registry being passed doesn't require authentication for pull actions + registry: + base_url: docker.io + # The owner or project of the registry + # For example, "reedcompbio" if the image is available as docker.io/reedcompbio/allpairs + owner: reedcompbio algorithms: - name: pathlinker diff --git a/test/generate-inputs/inputs/test_config.yaml b/test/generate-inputs/inputs/test_config.yaml index ed59497b1..0c83017fe 100644 --- a/test/generate-inputs/inputs/test_config.yaml +++ b/test/generate-inputs/inputs/test_config.yaml @@ -1,9 +1,10 @@ hash_length: 7 -container_framework: docker -unpack_singularity: false -container_registry: - base_url: docker.io - owner: reedcompbio +containers: + framework: docker + unpack_singularity: false + registry: + base_url: docker.io + owner: reedcompbio algorithms: - name: "pathlinker" diff --git a/test/test_config.py b/test/test_config.py index f5ec454b7..c8b05f3c5 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -18,10 +18,12 @@ # individual values of the dict can be changed and the whole initialization can be re-run. def get_test_config(): test_raw_config = { - "container_framework": "singularity", - "container_registry": { - "base_url": "docker.io", - "owner": "reedcompbio", + "containers": { + "framework": "singularity", + "registry": { + "base_url": "docker.io", + "owner": "reedcompbio", + }, }, "hash_length": 7, "reconstruction_settings": { @@ -159,46 +161,46 @@ def test_config_container_framework_normalization(self): # Test singularity test_config = get_test_config() - test_config["container_framework"] = "singularity" + test_config["containers"]["framework"] = "singularity" config.init_global(test_config) - assert (config.config.container_framework == "singularity") + assert (config.config.container_settings.framework == "singularity") # Test singularity with capitalization - test_config["container_framework"] = "Singularity" + test_config["containers"]["framework"] = "Singularity" config.init_global(test_config) - assert (config.config.container_framework == "singularity") + assert (config.config.container_settings.framework == "singularity") # Test docker - test_config["container_framework"] = "docker" + test_config["containers"]["framework"] = "docker" config.init_global(test_config) - assert (config.config.container_framework == "docker") + assert (config.config.container_settings.framework == "docker") # Test docker with capitalization - test_config["container_framework"] = "Docker" + test_config["containers"]["framework"] = "Docker" config.init_global(test_config) - assert (config.config.container_framework == "docker") + assert (config.config.container_settings.framework == "docker") # Test unknown framework - test_config["container_framework"] = "badFramework" + test_config["containers"]["framework"] = "badFramework" with pytest.raises(ValueError): config.init_global(test_config) def test_config_container_registry(self): test_config = get_test_config() - test_config["container_registry"]["base_url"] = "docker.io" - test_config["container_registry"]["owner"] = "reedcompbio" + test_config["containers"]["registry"]["base_url"] = "docker.io" + test_config["containers"]["registry"]["owner"] = "reedcompbio" config.init_global(test_config) - assert (config.config.container_prefix == "docker.io/reedcompbio") + assert (config.config.container_settings.prefix == "docker.io/reedcompbio") - test_config["container_registry"]["base_url"] = "another.repo" - test_config["container_registry"]["owner"] = "different-owner" + test_config["containers"]["registry"]["base_url"] = "another.repo" + test_config["containers"]["registry"]["owner"] = "different-owner" config.init_global(test_config) - assert (config.config.container_prefix == "another.repo/different-owner") + assert (config.config.container_settings.prefix == "another.repo/different-owner") - test_config["container_registry"]["base_url"] = "" - test_config["container_registry"]["owner"] = "" + test_config["containers"]["registry"]["base_url"] = "" + test_config["containers"]["registry"]["owner"] = "" config.init_global(test_config) - assert (config.config.container_prefix == config.DEFAULT_CONTAINER_PREFIX) + assert (config.config.container_settings.prefix == config.DEFAULT_CONTAINER_PREFIX) def test_error_dataset_label(self): test_config = get_test_config()