diff --git a/.github/workflows/httomo_docs.yml b/.github/workflows/httomo_docs.yml index 62e0f0c46..13c56b93d 100644 --- a/.github/workflows/httomo_docs.yml +++ b/.github/workflows/httomo_docs.yml @@ -35,9 +35,9 @@ jobs: run: | pip install --no-deps httomo-backends - - name: Generate full yaml pipelines using directives + - name: Generate full yaml pipelines using pipeline directives run: | - python ./docs/source/scripts/yaml_pipelines_generator.py -i ./docs/source/pipelines_full/gpu_pipeline1_directive.yaml -o ./docs/source/pipelines_full/gpu_pipeline1.yaml + python ./docs/source/scripts/execute_pipelines_build.py -o ./docs/source/pipelines_full/ - name: Build docs run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/ diff --git a/.github/workflows/run_tests_iris.yml b/.github/workflows/run_tests_framework_iris.yml similarity index 75% rename from .github/workflows/run_tests_iris.yml rename to .github/workflows/run_tests_framework_iris.yml index c107a8513..0501d6e65 100644 --- a/.github/workflows/run_tests_iris.yml +++ b/.github/workflows/run_tests_framework_iris.yml @@ -1,4 +1,4 @@ -name: HTTomo tests +name: HTTomo framework tests on: pull_request: @@ -38,9 +38,14 @@ jobs: pip install --upgrade --force-reinstall pillow pip install httomolibgpu tomobar pip install --no-deps httomo-backends - pip install . + pip install . micromamba list - - name: Run HTTomo tests + - name: Generate full yaml pipelines using pipeline directives + run: | + pip install "ruamel.yaml>0.18.0" + python ./docs/source/scripts/execute_pipelines_build.py -o ./docs/source/pipelines_full/ + + - name: Run HTTomo framework tests run: | pytest tests/ diff --git a/.github/workflows/run_tests_pipelines_iris.yml b/.github/workflows/run_tests_pipelines_iris.yml new file mode 100644 index 000000000..d72e0ca21 --- /dev/null +++ b/.github/workflows/run_tests_pipelines_iris.yml @@ -0,0 +1,51 @@ +name: HTTomo pipelines tests + +on: + pull_request: + branches: + - main + push: + branches: + - main + +jobs: + iris-gpu: + runs-on: iris-gpu + container: + image: nvidia/cuda:12.6.3-devel-ubi8 + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + + defaults: + run: + shell: bash -l {0} + + steps: + - name: Checkout repository code + uses: actions/checkout@v4 + + - name: Create conda environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: conda/environment.yml + environment-name: httomo + post-cleanup: 'all' + init-shell: bash + + - name: Install httomo libraries, httomo-backends and httomo + run: | + micromamba activate httomo + pip install --upgrade --force-reinstall pillow + pip install httomolibgpu tomobar + pip install --no-deps httomo-backends + pip install . + micromamba list + + - name: Generate full yaml pipelines using pipeline directives + run: | + pip install "ruamel.yaml>0.18.0" + python ./docs/source/scripts/execute_pipelines_build.py -o ./docs/source/pipelines_full/ + + - name: Run HTTomo pipelines tests (small data) + run: | + pytest tests/test_pipeline_small.py --small_data diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml index dfeaf21eb..3c1ac82a4 100644 --- a/docs/source/doc-conda-requirements.yml +++ b/docs/source/doc-conda-requirements.yml @@ -4,11 +4,11 @@ channels: dependencies: - python>=3.10,<3.13 - numpy - - sphinx + - sphinx>=8.0,<8.2.0 - sphinx-book-theme - - nbsphinx - pandoc - jinja2 + - nbsphinx - sphinx-design - sphinx-copybutton - pyyaml diff --git a/docs/source/pipelines/yaml.rst b/docs/source/pipelines/yaml.rst index 62fb8f1f1..416b50fcd 100644 --- a/docs/source/pipelines/yaml.rst +++ b/docs/source/pipelines/yaml.rst @@ -3,70 +3,66 @@ Full YAML pipelines ============================== -This is a collection of ready to be used pipeline templates aka process lists for HTTomo. +This is a collection of ready to be used full pipelines or process lists for HTTomo. See more on :ref:`explanation_process_list` and how to :ref:`howto_process_list`. -.. _tutorials_pl_templates_cpu: +HTTomo mainly targets GPU computations, therefore the use of :ref:`tutorials_pl_templates_gpu` is +preferable. However, when the GPU device is not available or a GPU method is not implemented, the use of +:ref:`tutorials_pl_templates_cpu` is possible. -CPU Pipeline templates ----------------------------- +.. note:: The combination of both GPU and CPU methods is possible. If one expects to achieve the faster performance, please use the GPU methods provided, where possible. -CPU-pipelines mostly use TomoPy methods that are executed on the CPU and expected to be slower. +.. _tutorials_pl_templates_gpu: -.. dropdown:: Basic TomoPy's (CPU-only) pipeline for the classical 180-degrees scan +GPU Pipeline templates +----------------------- - .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_cpu1.yaml - :language: yaml +The GPU-pipelines consist of methods from httomolibgpu (GPU) and httomolib (CPU) backend :ref:`backends_list`. Those libraries are supported directly by the HTTomo development team. -.. dropdown:: TomoPy's pipeline where :ref:`previewing` is demonstrated +.. dropdown:: GPU pipeline with auto-centering and the FBP reconstruction method. - .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_cpu2.yaml + .. literalinclude:: ../pipelines_full/gpu_pipelineFBP.yaml :language: yaml -.. dropdown:: This pipeline shows how "calculate_stats" module extracts global statistics in order to rescale data for saving 8-bit images +.. dropdown:: GPU pipeline as above and Total Variation denoising on the result of the FBP reconstruction. - .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_cpu3.yaml + .. literalinclude:: ../pipelines_full/gpu_pipelineFBP_denoising.yaml :language: yaml -.. _tutorials_pl_templates_gpu: +.. _tutorials_pl_templates_cpu: -GPU Pipeline templates ----------------------------- +CPU Pipeline templates +----------------------- -It is recommended to use GPU-based pipelines and methods from the httomolib and httomolibgpu libraries. Those libraries are supported directly by HTTomo development team. +The CPU-pipelines mostly use TomoPy methods. They are executed solely on the CPU and therefore expected to be slower than the GPU pipelines. -.. dropdown:: Basic GPU pipeline which uses functions from the httomolibgpu library. +.. dropdown:: CPU pipeline using auto-centering and the gridrec reconstruction method on the CPU (TomoPy). - .. literalinclude:: ../pipelines_full/gpu_pipeline1.yaml + .. literalinclude:: ../pipelines_full/cpu_pipeline_gridrec.yaml :language: yaml .. _tutorials_pl_templates_dls: DLS-specific templates ----------------------------- +---------------------- -Those pipelines will use the methods from the httomolib and httomolibgpu libraries. - -.. dropdown:: An example of a typical DIAD (k11) beamline piepeline. - - .. literalinclude:: ../../../tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml - :language: yaml +Those pipelines are specific to Diamond Light Source processing strategies and can vary between different tomographic beamlines. -.. dropdown:: Pipeline for 360-degrees data with automatic CoR finding and stitching to 180-degrees data. +.. dropdown:: An example of DIAD-k11 beamline pipeline with auto-centering and FBP reconstruction on the GPU. - .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml + .. literalinclude:: ../pipelines_full/gpu_diad_FBP.yaml :language: yaml -.. dropdown:: Pipeline for 360-degrees data with automatic CoR finding and stitching to 180-degrees data. Iterative reconstruction +.. dropdown:: Pipeline for 360-degrees data with automatic CoR/overlap finding and stitching to 180-degrees data. - .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_360deg_iterative_gpu3.yaml + .. literalinclude:: ../pipelines_full/gpu_360_paganin_FBP.yaml :language: yaml .. _tutorials_pl_templates_sweeps: Parameter Sweeps templates ----------------------------- +-------------------------- These templates demonstrate how to perform a sweep across multiple values of a single parameter (see :ref:`parameter_sweeping` for more details). diff --git a/docs/source/pipelines_full/__init__.py b/docs/source/pipelines_full/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/pipelines_full/gpu_pipeline1_directive.yaml b/docs/source/pipelines_full/gpu_pipeline1_directive.yaml deleted file mode 100644 index 9b6d2533a..000000000 --- a/docs/source/pipelines_full/gpu_pipeline1_directive.yaml +++ /dev/null @@ -1,18 +0,0 @@ -- method: standard_tomo - module_path: httomo.data.hdf.loaders -- method: find_center_vo - module_path: httomolibgpu.recon.rotation -- method: remove_outlier - module_path: httomolibgpu.misc.corr -- method: normalize - module_path: httomolibgpu.prep.normalize -- method: remove_all_stripe - module_path: httomolibgpu.prep.stripe -- method: FBP - module_path: httomolibgpu.recon.algorithm -- method: calculate_stats - module_path: httomo.methods -- method: rescale_to_int - module_path: httomolibgpu.misc.rescale -- method: save_to_images - module_path: httomolib.misc.images diff --git a/docs/source/scripts/execute_pipelines_build.py b/docs/source/scripts/execute_pipelines_build.py new file mode 100644 index 000000000..d14477b80 --- /dev/null +++ b/docs/source/scripts/execute_pipelines_build.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------- +# Copyright 2022 Diamond Light Source Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +# Created By : Tomography Team +# Created Date: 30/January/2025 +# version ='0.1' +# --------------------------------------------------------------------------- +"""Executing full-pipeline generation for HTTomo using YAML templates from httomo-backends +and yaml_pipelines_generator script available also in httomo-backends. +""" + +import argparse +import os +import glob +import httomo_backends +from httomo_backends.scripts.yaml_pipelines_generator import yaml_pipelines_generator + + +def get_args(): + parser = argparse.ArgumentParser( + description="Script that generates YAML pipelines for HTTomo " + "using YAML templates from httomo-backends." + ) + parser.add_argument( + "-o", + "--output", + type=str, + default="./", + help="Full path to the output pipelines folder.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + path_to_httomobackends = os.path.dirname(httomo_backends.__file__) + args = get_args() + path_to_httomo_pipelines = args.output + pipelines_folder = path_to_httomobackends + "/pipelines_full/" + # loop over all pipeline directive files and running the generator + for filepath in glob.iglob(pipelines_folder + "*.yaml"): + basename = os.path.basename(filepath) + outputfile_name = os.path.normpath(basename.replace(r"_directive", r"")) + yaml_pipelines_generator( + filepath, path_to_httomobackends, path_to_httomo_pipelines + outputfile_name + ) + message_str = f"{outputfile_name} has been generated." + print(message_str) diff --git a/docs/source/scripts/yaml_pipelines_generator.py b/docs/source/scripts/yaml_pipelines_generator.py deleted file mode 100644 index 0892463d4..000000000 --- a/docs/source/scripts/yaml_pipelines_generator.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------- -# Copyright 2022 Diamond Light Source Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- -# Created By : Tomography Team -# Created Date: 22/January/2025 -# version ='0.1' -# --------------------------------------------------------------------------- -"""Script that generates YAML pipeline for HTTomo using YAML templates from httomo-backends -(should be installed in environment). - -Please run the generator as: - python -m yaml_pipelines_generator -i /path/to/pipelines.yml -o /path/to/output/ -""" -import argparse -import os -import ruamel.yaml -import httomo_backends - -CS = ruamel.yaml.comments.CommentedSeq # defaults to block style - - -def __represent_none(self, data): - return self.represent_scalar("tag:yaml.org,2002:null", "null") - - -def yaml_pipelines_generator( - path_to_pipelines: str, path_to_httomobackends: str, path_to_output_file: str -) -> int: - """function that builds YAML pipeline using YAML templates from httomo-backends - - Args: - path_to_pipelines: path to the YAML file which contains a high-level description of the required pipeline to be built. - path_to_httomobackends: path to httomo-backends on the system, where YAML templates stored. - path_to_output_file: path to output file with the generated pipeline - - Returns: - returns zero if the processing is successful - """ - - yaml = ruamel.yaml.YAML(typ="rt", pure=True) - - # open YAML file to inspect - with open(path_to_pipelines, "r") as file: - try: - pipeline_file_content = yaml.load(file) - except OSError as e: - print("loading yaml file with methods failed", e) - - with open(path_to_output_file, "w") as f: - # a loop over methods in the high-level pipeline file (directive) - methods_no = len(pipeline_file_content) - pipeline_full = CS() - for i in range(methods_no): - method_content = pipeline_file_content[i] - method_name = method_content["method"] - module_name = method_content["module_path"] - # get the corresponding yaml template from httomo-backends - backend_name = module_name[0 : module_name.find(".")] - full_path_to_yamls = ( - path_to_httomobackends - + "/yaml_templates/" - + backend_name - + "/" - + module_name - + "/" - + method_name - + ".yaml" - ) - with open(full_path_to_yamls, "r") as stream: - try: - yaml_template_method = yaml.load(stream) - except OSError as e: - print("loading yaml template failed", e) - - if "loaders" in module_name: - # should be the first method in the list - pipeline_full.yaml_set_comment_before_after_key( - i, - "--- Standard tomography loader for NeXus files. ---", - indent=0, - ) - pipeline_full += yaml_template_method - elif "rotation" in module_name: - pipeline_full.yaml_set_comment_before_after_key( - i, - "--- Center of Rotation auto-finding. Required for reconstruction bellow. ---", - indent=0, - ) - pipeline_full += yaml_template_method - pipeline_full[i]["parameters"].yaml_add_eol_comment( - key="ind", - comment="A vertical slice (sinogram) index to calculate CoR, `mid` can be used for middle", - ) - pipeline_full[i]["parameters"].yaml_add_eol_comment( - key="cor_initialisation_value", - comment="Use if an approximate CoR is known", - ) - pipeline_full[i]["parameters"].yaml_add_eol_comment( - key="average_radius", - comment="Average several sinograms to improve SNR, one can try 3-5 range", - ) - pipeline_full[i]["side_outputs"].yaml_add_eol_comment( - key="cor", - comment="A side output of the method, here a CoR scalar value", - ) - elif "corr" in module_name and "remove_outlier" in method_name: - pipeline_full.yaml_set_comment_before_after_key( - i, - "--- Removing dead pixels in the data, aka zingers. Use if sharp streaks are present in reconstruction. ---", - indent=0, - ) - pipeline_full += yaml_template_method - pipeline_full[i]["parameters"].yaml_add_eol_comment( - key="dif", - comment="A difference between the outlier value and the median value of neighbouring pixels.", - ) - elif "normalize" in module_name: - pipeline_full.yaml_set_comment_before_after_key( - i, - "--- Normalisation of projection data using collected flats/darks images. --- ", - indent=0, - ) - pipeline_full += yaml_template_method - pipeline_full[i]["parameters"].yaml_add_eol_comment( - key="minus_log", - comment="If Paganin method is used bellow, set it to false.", - ) - elif "stripe" in module_name: - pipeline_full.yaml_set_comment_before_after_key( - i, - "--- Method to remove stripe artefacts in the data that lead to ring artefacts in the reconstruction. --- ", - indent=0, - ) - pipeline_full += yaml_template_method - elif "algorithm" in module_name: - pipeline_full.yaml_set_comment_before_after_key( - i, - "--- Reconstruction method. ---", - indent=0, - ) - pipeline_full += yaml_template_method - pipeline_full[i]["parameters"].yaml_add_eol_comment( - key="center", - comment="Reference to center of rotation side output OR an integer.", - ) - pipeline_full[i]["parameters"].yaml_add_eol_comment( - key="recon_mask_radius", - comment="Zero pixels outside the mask-circle radius.", - ) - elif "calculate_stats" in method_name: - pipeline_full.yaml_set_comment_before_after_key( - i, - "--- Calculate global statistics on the reconstructed volume, required for data rescaling. ---", - indent=0, - ) - pipeline_full += yaml_template_method - elif "rescale_to_int" in method_name: - pipeline_full.yaml_set_comment_before_after_key( - i, - "--- Rescaling the data using min/max obtained from `calculate_stats`. ---", - indent=0, - ) - pipeline_full += yaml_template_method - elif "images" in module_name: - pipeline_full.yaml_set_comment_before_after_key( - i, - "--- Saving data into images. ---", - indent=0, - ) - pipeline_full += yaml_template_method - pipeline_full[i]["parameters"].yaml_add_eol_comment( - key="file_format", - comment="`tif` or `jpeg` can be used.", - ) - else: - pipeline_full.yaml_set_comment_before_after_key( - i, - "--------------------------------------------------------#", - indent=0, - ) - pipeline_full += yaml_template_method - - yaml.representer.add_representer(type(None), __represent_none) - yaml.dump(pipeline_full, f) - - return 0 - - -def get_args(): - parser = argparse.ArgumentParser( - description="Script that generates YAML pipelines for HTTomo " - "using YAML templates from httomo-backends." - ) - parser.add_argument( - "-i", - "--input", - type=str, - default=None, - help="A path to the list of pipelines needed to be built within a yaml file", - ) - parser.add_argument( - "-o", - "--output", - type=str, - default="./", - help="Full path to the yaml file with the generated pipeline.", - ) - return parser.parse_args() - - -if __name__ == "__main__": - path_to_httomobackends = os.path.dirname(httomo_backends.__file__) - args = get_args() - path_to_pipelines = args.input - path_to_output_file = args.output - return_val = yaml_pipelines_generator( - path_to_pipelines, path_to_httomobackends, path_to_output_file - ) - if return_val == 0: - print("YAML pipeline has been successfully generated!") diff --git a/tests/conftest.py b/tests/conftest.py index ad7dffa1c..e6c47e484 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ import sys from pathlib import Path from shutil import rmtree -from typing import Any, Callable, Dict, List, TypeAlias +from typing import Any, Callable, Dict, List, TypeAlias, Union import numpy as np import pytest @@ -24,6 +24,12 @@ def pytest_configure(config): config.addinivalue_line("markers", "mpi: mark test to run in an MPI environment") config.addinivalue_line("markers", "perf: mark test as performance test") config.addinivalue_line("markers", "cupy: needs cupy to run") + config.addinivalue_line( + "markers", "small_data: mark tests to run full pipelines on small data" + ) + config.addinivalue_line( + "markers", "full_data: mark tests to run full pipelines on raw big data" + ) config.addinivalue_line( "markers", "preview: mark test to run with `httomo preview`" ) @@ -36,6 +42,18 @@ def pytest_addoption(parser): default=False, help="run performance tests only", ) + parser.addoption( + "--small_data", + action="store_true", + default=False, + help="run full pipelines on small data", + ) + parser.addoption( + "--full_data", + action="store_true", + default=False, + help="run full pipelines on raw (big) data", + ) def pytest_collection_modifyitems(config, items): @@ -51,6 +69,30 @@ def pytest_collection_modifyitems(config, items): for item in items: if "perf" in item.keywords: item.add_marker(skip_perf) + if config.getoption("--small_data"): + skip_other = pytest.mark.skip(reason="not a pipeline small data test") + for item in items: + if "small_data" not in item.keywords: + item.add_marker(skip_other) + else: + skip_perf = pytest.mark.skip( + reason="pipeline small data test - use '--small_data' to run" + ) + for item in items: + if "small_data" in item.keywords: + item.add_marker(skip_perf) + if config.getoption("--full_data"): + skip_other = pytest.mark.skip(reason="not a pipeline raw big data test") + for item in items: + if "full_data" not in item.keywords: + item.add_marker(skip_other) + else: + skip_perf = pytest.mark.skip( + reason="pipeline raw big data test - use '--full_data' to run" + ) + for item in items: + if "full_data" in item.keywords: + item.add_marker(skip_perf) @pytest.fixture @@ -174,98 +216,167 @@ def standard_image_key_path(): return "/entry1/tomo_entry/instrument/detector/image_key" -@pytest.fixture -def testing_pipeline(): - return "tests/samples/pipeline_template_examples/testing/testing_pipeline.yaml" - - +# TODO: depricate when loader is generalised (big data tests instead) @pytest.fixture def diad_data(): return "tests/test_data/k11_diad/k11-18014.nxs" +# TODO: depricate when loader is generalised @pytest.fixture def diad_loader(): return "tests/samples/loader_configs/diad.yaml" @pytest.fixture -def diad_pipeline_gpu(): - return "tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml" +def standard_loader(): + return "tests/samples/loader_configs/standard_tomo.yaml" @pytest.fixture -def i12_data(): - return "tests/test_data/i12/separate_flats_darks/i12_dynamic_start_stop180.nxs" +def sample_pipelines(): + return "tests/samples/pipeline_template_examples/" + + +#####################Auto-generated pipelines################## @pytest.fixture -def pipeline360(): - return "samples/pipeline_template_examples/DLS/02_i12_360scan_pipeline.yaml" +def cpu_pipeline_gridrec(): + return "docs/source/pipelines_full/cpu_pipeline_gridrec.yaml" @pytest.fixture -def i12_loader(): - return ( - "tests/samples/pipeline_template_examples/DLS/03_i12_separate_darks_flats.yaml" - ) +def gpu_pipelineFBP(): + return "docs/source/pipelines_full/gpu_pipelineFBP.yaml" @pytest.fixture -def i12_loader_ignore_darks_flats(): - return "tests/samples/pipeline_template_examples/DLS/04_i12_ignore_darks_flats.yaml" +def gpu_pipelineFBP_denoising(): + return "docs/source/pipelines_full/gpu_pipelineFBP_denoising.yaml" @pytest.fixture -def standard_loader(): - return "tests/samples/loader_configs/standard_tomo.yaml" +def gpu_pipeline_diad_FBP_noimagesaving(): + return "docs/source/pipelines_full/gpu_diad_FBP_noimagesaving.yaml" @pytest.fixture -def sample_pipelines(): - return "tests/samples/pipeline_template_examples/" +def gpu_pipeline_diad_FBP(): + return "docs/source/pipelines_full/gpu_diad_FBP.yaml" @pytest.fixture -def gpu_pipeline(): - return "tests/samples/pipeline_template_examples/03_basic_gpu_pipeline_tomo_standard.yaml" +def gpu_pipeline_360_paganin_FBP(): + return "docs/source/pipelines_full/gpu_360_paganin_FBP.yaml" + + +# ---------------------END------------------------# + +###########Raw projection data (large)################## + +# The fixtures bellow exist for the testing of HTTomo with raw projection data at Diamond. +# Jenkins CI at Diamond will provide an access to this test data. Otherwise +# this test data is not available to the user and the relevant tests that are marked as `full_data` +# will be ignored. + + +@pytest.fixture +def diad_k11_38727(): + # 4k projections, 45gb dataset + return "tests/test_data/raw_data/diad/k11-38727.nxs" + + +@pytest.fixture +def diad_k11_38729(): + # 2k projections, 22gb dataset + return "tests/test_data/raw_data/diad/k11-38729.nxs" + + +@pytest.fixture +def diad_k11_38730(): + # 1k projections, 11gb dataset + return "tests/test_data/raw_data/diad/k11-38730.nxs" + + +@pytest.fixture +def diad_k11_38731(): + # 0.5k projections, 6gb dataset + return "tests/test_data/raw_data/diad/k11-38731.nxs" + + +@pytest.fixture +def i13_177906(): + # 2.5k projections, 27gb dataset + return "tests/test_data/raw_data/i13/177906.nxs" + + +@pytest.fixture +def i13_179623(): + # 6k projections, 65gb dataset, 360 degrees scan + return "tests/test_data/raw_data/i13/360/179623.nxs" + + +############## --Ground Truth references-- ################# + + +@pytest.fixture +def gpu_diad_FBP_k11_38731_npz(): + # 10 slices numpy array + return np.load("tests/test_data/raw_data/diad/gpu_diad_FBP_k11-38731.npz") @pytest.fixture -def yaml_cpu_pipeline1(): - return "tests/samples/pipeline_template_examples/pipeline_cpu1.yaml" +def gpu_diad_FBP_k11_38730_npz(): + # 10 slices numpy array + return np.load("tests/test_data/raw_data/diad/gpu_diad_FBP_k11-38730.npz") @pytest.fixture -def yaml_cpu_pipeline2(): - return "tests/samples/pipeline_template_examples/pipeline_cpu2.yaml" +def gpu_FBP_TVdenoising_i13_177906_npz(): + # 10 slices numpy array + return np.load("tests/test_data/raw_data/i13/gpu_FBP_TVdenoising_i13_177906.npz") @pytest.fixture -def yaml_cpu_pipeline3(): - return "tests/samples/pipeline_template_examples/pipeline_cpu3.yaml" +def gpu_FBP_paganin_i13_179623_npz(): + # 10 slices numpy array + return np.load("tests/test_data/raw_data/i13/360/gpu_FBP_paganin_i13_179623.npz") + + +# ---------------------END------------------------# +# TODO: deprecate when loader is generalised @pytest.fixture -def yaml_cpu_pipeline4(): - return "tests/samples/pipeline_template_examples/pipeline_cpu4.yaml" +def diad_pipeline_gpu(): + return "tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml" @pytest.fixture -def yaml_cpu_pipeline5(): - return "tests/samples/pipeline_template_examples/pipeline_cpu5.yaml" +def i12_data(): + return "tests/test_data/i12/separate_flats_darks/i12_dynamic_start_stop180.nxs" +# TODO: move to big pipeline tests @pytest.fixture -def yaml_gpu_pipeline1(): - return "tests/samples/pipeline_template_examples/pipeline_gpu1.yaml" +def pipeline360(): + return "samples/pipeline_template_examples/DLS/02_i12_360scan_pipeline.yaml" @pytest.fixture -def yaml_gpu_pipeline360_2(): - return "tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml" +def i12_loader(): + return ( + "tests/samples/pipeline_template_examples/DLS/03_i12_separate_darks_flats.yaml" + ) + + +@pytest.fixture +def i12_loader_ignore_darks_flats(): + return "tests/samples/pipeline_template_examples/DLS/04_i12_ignore_darks_flats.yaml" +###########Sweep pipelines (not autogenerated currently)############### @pytest.fixture def yaml_gpu_pipeline_sweep_cor(): return "tests/samples/pipeline_template_examples/parameter-sweep-cor.yaml" @@ -276,6 +387,9 @@ def yaml_gpu_pipeline_sweep_paganin(): return "tests/samples/pipeline_template_examples/parameter-sweep-paganin.yaml" +# ---------------------END------------------------# + + @pytest.fixture(scope="session") def distortion_correction_path(test_data_path): return os.path.join(test_data_path, "distortion-correction") @@ -360,3 +474,33 @@ def _load_yaml(yaml_in: str) -> PipelineConfig: return conf[0] return _load_yaml + + +def change_value_parameters_method_pipeline( + yaml_path: str, + method: list, + key: list, + value: list, + save_result: Union[None, bool] = None, +): + # changes methods parameters in the given pipeline and re-save the pipeline + with open(yaml_path, "r") as f: + conf = list(yaml.load_all(f, Loader=yaml.FullLoader)) + opened_yaml = conf[0] + methods_no = len(opened_yaml) + methods_no_correct = len(method) + for i in range(methods_no): + method_content = opened_yaml[i] + method_name = method_content["method"] + for j in range(methods_no_correct): + if method[j] == method_name: + # change something in parameters here + opened_yaml[i]["parameters"][key[j]] = value[j] + if save_result is not None: + # add save_result to the list of keys + opened_yaml[i]["save_result"] = save_result + + with open(yaml_path, "w") as file_descriptor: + yaml.dump( + opened_yaml, file_descriptor, default_flow_style=False, sort_keys=False + ) diff --git a/tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml b/tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml deleted file mode 100644 index 404783166..000000000 --- a/tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# Standard tomography loader for NeXus files -#-----------------------------------------------# -- method: standard_tomo - module_path: httomo.data.hdf.loaders - parameters: - data_path: /entry/imaging/data - image_key_path: /entry/instrument/imaging/image_key - rotation_angles: - data_path: /entry/imaging_sum/gts_theta_value - preview: - detector_x: # horizontal data previewing/cropping - start: - stop: - detector_y: # vertical data previewing/cropping - start: 8 - stop: 15 -# Center of Rotation method for automatic centering. Required for reconstruction. -#-----------------------------------------------# -- method: find_center_vo - module_path: httomolibgpu.recon.rotation - parameters: # see online documentation for parameters - ind: mid # specify the vertical index (slice) for calculation. mid - middle - smin: -50 - smax: 50 - srad: 6 - step: 0.25 - ratio: 0.5 - drop: 20 - id: centering # method's id for future referencing (see reconstruction) - side_outputs: # method's side outputs include scalars and/or some auxiliary data output - cor: centre_of_rotation -# Remove dezingers (outliers) in the data -#-----------------------------------------------# -- method: remove_outlier - module_path: httomolibgpu.misc.corr - parameters: - dif: 0.1 # this might require optimisation - kernel_size: 3 -# Normalisation of projection data with collected flats/darks. -#-----------------------------------------------# -- method: normalize - module_path: httomolibgpu.prep.normalize - parameters: - cutoff: 10.0 - minus_log: true # set to false if Paganin method is used - nonnegativity: false - remove_nans: false -# Remove stripes in the data that can lead to ring artefacts in reconstruction -#-----------------------------------------------# -- method: remove_stripe_based_sorting - module_path: httomolibgpu.prep.stripe - parameters: - size: 11 - dim: 1 -# Reconstruction method -#-----------------------------------------------# -- method: FBP - module_path: httomolibgpu.recon.algorithm - parameters: - center: ${{centering.side_outputs.centre_of_rotation}} # the reference to the found CoR. Manually found integer can be also used. - filter_freq_cutoff: 0.35 - recon_size: null - recon_mask_radius: 0.95 - save_result: true # set to false if hdf5 is not needed -# Calculate global statistics on the reconstructed volume (min/max needed specifically) -#-----------------------------------------------# -- method: calculate_stats - module_path: httomo.methods - parameters: {} - id: statistics - side_outputs: - glob_stats: glob_stats -# Rescaling the data into 8-bit unsigned integer for saving into tiffs -#-----------------------------------------------# -- method: rescale_to_int - module_path: httomolibgpu.misc.rescale - parameters: - perc_range_min: 5.0 - perc_range_max: 95.0 - bits: 8 - glob_stats: ${{statistics.side_outputs.glob_stats}} # referring to min/max values of statistics -# Saving the rescaled data into tiffs -#-----------------------------------------------# -- method: save_to_images - module_path: httomolib.misc.images - parameters: - subfolder_name: images - axis: auto - file_format: tif diff --git a/tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml b/tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml deleted file mode 100644 index 46e8309bd..000000000 --- a/tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Standard tomography loader for NeXus files -#-----------------------------------------------# -- method: standard_tomo - module_path: httomo.data.hdf.loaders - parameters: - data_path: entry1/tomo_entry/data/data - image_key_path: entry1/tomo_entry/instrument/detector/image_key - rotation_angles: - data_path: /entry1/tomo_entry/data/rotation_angle -# Automatic search for the centre of rotation and overlap to perform 360 to 180 degrees data stitching -#-----------------------------------------------# -- method: find_center_360 - module_path: httomolibgpu.recon.rotation - parameters: - ind: mid # specify the vertical index (slice) for calculation. mid - middle - win_width: 10 - side: null - denoise: true - norm: false - use_overlap: false - id: centering # method's id for future referencing (see reconstruction) - side_outputs: # method's side outputs include scalars and/or some auxiliary data output - cor: centre_of_rotation - overlap: overlap - side: side - overlap_position: overlap_position -# Remove dezingers (outliers) in the data -#-----------------------------------------------# -- method: remove_outlier - module_path: httomolibgpu.misc.corr - parameters: - dif: 0.1 # this might require optimisation - kernel_size: 3 -# Normalisation of projection data with collected flats/darks. -#-----------------------------------------------# -- method: normalize - module_path: httomolibgpu.prep.normalize - parameters: - cutoff: 10.0 - minus_log: true # set to false if Paganin method is used - nonnegativity: false - remove_nans: false -# Remove stripes in the data that can lead to ring artefacts in reconstruction -#-----------------------------------------------# -- method: remove_stripe_based_sorting - module_path: httomolibgpu.prep.stripe - parameters: - size: 11 - dim: 1 -# Convert 360 degrees data into 180 degrees data -#-----------------------------------------------# -- method: sino_360_to_180 - module_path: httomolibgpu.misc.morph - parameters: - overlap: ${{centering.side_outputs.overlap}} # the reference to the found CoR. - rotation: right -- method: FBP - module_path: httomolibgpu.recon.algorithm - parameters: - center: ${{centering.side_outputs.centre_of_rotation}} - filter_freq_cutoff: 0.35 - recon_size: null - recon_mask_radius: 0.95 - save_result: true # save the hdf5 array -# Downsample the reconstructed data to smaller data -#-----------------------------------------------# -- method: data_resampler - module_path: httomolibgpu.misc.morph - parameters: - newshape: [500, 500] - axis: auto - interpolation: linear -# Calculate global statistics on the reconstructed volume (min/max needed specifically) -#-----------------------------------------------# -- method: calculate_stats - module_path: httomo.methods - parameters: {} - id: statistics - side_outputs: - glob_stats: glob_stats -# Rescaling the data into 8-bit unsigned integer for saving into tiffs -#-----------------------------------------------# -- method: rescale_to_int - module_path: httomolibgpu.misc.rescale - parameters: - perc_range_min: 5.0 - perc_range_max: 95.0 - bits: 8 - glob_stats: ${{statistics.side_outputs.glob_stats}} # referring to min/max values of statistics -# Saving the rescaled data into tiffs -#-----------------------------------------------# -- method: save_to_images - module_path: httomolib.misc.images - parameters: - subfolder_name: images - axis: auto - file_format: tif diff --git a/tests/samples/pipeline_template_examples/pipeline_360deg_iterative_gpu3.yaml b/tests/samples/pipeline_template_examples/pipeline_360deg_iterative_gpu3.yaml deleted file mode 100644 index 5422aaa10..000000000 --- a/tests/samples/pipeline_template_examples/pipeline_360deg_iterative_gpu3.yaml +++ /dev/null @@ -1,78 +0,0 @@ -- method: standard_tomo - module_path: httomo.data.hdf.loaders - parameters: - data_path: entry1/tomo_entry/data/data - image_key_path: entry1/tomo_entry/instrument/detector/image_key - rotation_angles: - data_path: /entry1/tomo_entry/data/rotation_angle - preview: - detector_y: - start: 1000 - stop: 1005 -- method: find_center_360 - module_path: httomolibgpu.recon.rotation - parameters: - ind: mid - win_width: 10 - side: null - denoise: true - norm: false - use_overlap: false - id: centering - side_outputs: - cor: centre_of_rotation - overlap: overlap - side: side - overlap_position: overlap_position -- method: normalize - module_path: httomolibgpu.prep.normalize - parameters: - cutoff: 10.0 - minus_log: true - nonnegativity: false - remove_nans: false -- method: sino_360_to_180 - module_path: httomolibgpu.misc.morph - parameters: - overlap: ${{centering.side_outputs.overlap}} - rotation: right -- method: remove_all_stripe - module_path: httomolibgpu.prep.stripe - parameters: - snr: 3.0 - la_size: 61 - sm_size: 21 - dim: 1 -- method: CGLS - module_path: httomolibgpu.recon.algorithm - parameters: - center: ${{centering.side_outputs.centre_of_rotation}} - recon_size: null - iterations: 3 - nonnegativity: true - save_result: true -- method: data_resampler - module_path: httomolibgpu.misc.morph - parameters: - newshape: [256, 256] - axis: auto - interpolation: nearest -- method: calculate_stats - module_path: httomo.methods - parameters: {} - id: statistics - side_outputs: - glob_stats: glob_stats -- method: rescale_to_int - module_path: httomolibgpu.misc.rescale - parameters: - perc_range_min: 0.0 - perc_range_max: 100.0 - bits: 8 - glob_stats: ${{statistics.side_outputs.glob_stats}} -- method: save_to_images - module_path: httomolib.misc.images - parameters: - subfolder_name: images - axis: auto - file_format: tif diff --git a/tests/samples/pipeline_template_examples/pipeline_cpu1.yaml b/tests/samples/pipeline_template_examples/pipeline_cpu1.yaml deleted file mode 100644 index 96ad90816..000000000 --- a/tests/samples/pipeline_template_examples/pipeline_cpu1.yaml +++ /dev/null @@ -1,47 +0,0 @@ -- method: standard_tomo - module_path: httomo.data.hdf.loaders - parameters: - data_path: entry1/tomo_entry/data/data - image_key_path: entry1/tomo_entry/instrument/detector/image_key - rotation_angles: - data_path: /entry1/tomo_entry/data/rotation_angle -- method: normalize - module_path: tomopy.prep.normalize - parameters: - cutoff: null - averaging: mean -- method: minus_log - module_path: tomopy.prep.normalize - parameters: {} -- method: find_center_vo - module_path: tomopy.recon.rotation - parameters: - ind: null - smin: -50 - smax: 50 - srad: 6 - step: 0.25 - ratio: 0.5 - drop: 20 - id: centering - side_outputs: - cor : centre_of_rotation -- method: recon - module_path: tomopy.recon.algorithm - parameters: - center: ${{centering.side_outputs.centre_of_rotation}} - sinogram_order: false - algorithm: 'gridrec' - init_recon: null -- method: rescale_to_int - module_path: httomolibgpu.misc.rescale - parameters: - perc_range_min: 0.0 - perc_range_max: 100.0 - bits: 8 -- method: save_to_images - module_path: httomolib.misc.images - parameters: - subfolder_name: images - axis: auto - file_format: tif diff --git a/tests/samples/pipeline_template_examples/pipeline_cpu2.yaml b/tests/samples/pipeline_template_examples/pipeline_cpu2.yaml deleted file mode 100644 index ef2600c2c..000000000 --- a/tests/samples/pipeline_template_examples/pipeline_cpu2.yaml +++ /dev/null @@ -1,71 +0,0 @@ -- method: standard_tomo - module_path: httomo.data.hdf.loaders - parameters: - data_path: entry1/tomo_entry/data/data - image_key_path: entry1/tomo_entry/instrument/detector/image_key - rotation_angles: - data_path: /entry1/tomo_entry/data/rotation_angle - preview: - detector_y: - start: 30 - stop: 60 -- method: find_center_vo - module_path: tomopy.recon.rotation - parameters: - ind: mid - smin: -50 - smax: 50 - srad: 6 - step: 0.25 - ratio: 0.5 - drop: 20 - id: centering - side_outputs: - cor: centre_of_rotation -- method: remove_outlier - module_path: tomopy.misc.corr - parameters: - dif: 0.1 - size: 3 - axis: auto -- method: normalize - module_path: tomopy.prep.normalize - parameters: - cutoff: null - averaging: mean -- method: minus_log - module_path: tomopy.prep.normalize - parameters: {} -- method: remove_stripe_fw - module_path: tomopy.prep.stripe - parameters: - level: null - wname: db5 - sigma: 2 - pad: true -- method: recon - module_path: tomopy.recon.algorithm - parameters: - center: ${{centering.side_outputs.centre_of_rotation}} - sinogram_order: false - algorithm: gridrec - init_recon: null - #additional parameters: AVAILABLE - save_result: true -- method: median_filter - module_path: tomopy.misc.corr - parameters: - size: 3 - axis: auto -- method: rescale_to_int - module_path: httomolibgpu.misc.rescale - parameters: - perc_range_min: 0.0 - perc_range_max: 100.0 - bits: 8 -- method: save_to_images - module_path: httomolib.misc.images - parameters: - subfolder_name: images - axis: auto - file_format: tif diff --git a/tests/samples/pipeline_template_examples/pipeline_cpu3.yaml b/tests/samples/pipeline_template_examples/pipeline_cpu3.yaml deleted file mode 100644 index 81331fd94..000000000 --- a/tests/samples/pipeline_template_examples/pipeline_cpu3.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- method: standard_tomo - module_path: httomo.data.hdf.loaders - parameters: - data_path: entry1/tomo_entry/data/data - image_key_path: entry1/tomo_entry/instrument/detector/image_key - rotation_angles: - data_path: /entry1/tomo_entry/data/rotation_angle -- method: find_center_vo - module_path: tomopy.recon.rotation - parameters: - ind: mid - smin: -50 - smax: 50 - srad: 6 - step: 0.25 - ratio: 0.5 - drop: 20 - id: centering - side_outputs: - cor: centre_of_rotation -- method: remove_outlier - module_path: tomopy.misc.corr - parameters: - dif: 0.1 - size: 3 - axis: auto -- method: normalize - module_path: tomopy.prep.normalize - parameters: - cutoff: null - averaging: mean -- method: minus_log - module_path: tomopy.prep.normalize - parameters: {} -- method: recon - module_path: tomopy.recon.algorithm - parameters: - center: ${{centering.side_outputs.centre_of_rotation}} - sinogram_order: false - algorithm: gridrec - init_recon: null - #additional parameters: AVAILABLE -- method: calculate_stats - module_path: httomo.methods - parameters: {} - id: statistics - side_outputs: - glob_stats: glob_stats -- method: rescale_to_int - module_path: httomolibgpu.misc.rescale - parameters: - perc_range_min: 0.0 - perc_range_max: 100.0 - bits: 8 - glob_stats: ${{statistics.side_outputs.glob_stats}} -- method: save_to_images - module_path: httomolib.misc.images - parameters: - subfolder_name: images - axis: auto - file_format: tif diff --git a/tests/samples/pipeline_template_examples/pipeline_cpu4.yaml b/tests/samples/pipeline_template_examples/pipeline_cpu4.yaml deleted file mode 100644 index f35ed2f09..000000000 --- a/tests/samples/pipeline_template_examples/pipeline_cpu4.yaml +++ /dev/null @@ -1,45 +0,0 @@ -- method: standard_tomo - module_path: httomo.data.hdf.loaders - parameters: - data_path: entry1/tomo_entry/data/data - image_key_path: entry1/tomo_entry/instrument/detector/image_key - rotation_angles: - data_path: /entry1/tomo_entry/data/rotation_angle -- method: find_center_pc - module_path: tomopy.recon.rotation - parameters: - proj1: auto - proj2: auto - tol: 0.5 - rotc_guess: null - id: centering - side_outputs: - cor: centre_of_rotation -- method: normalize - module_path: tomopy.prep.normalize - parameters: - cutoff: null - averaging: mean -- method: minus_log - module_path: tomopy.prep.normalize - parameters: {} -- method: recon - module_path: tomopy.recon.algorithm - parameters: - center: ${{centering.side_outputs.centre_of_rotation}} - sinogram_order: false - algorithm: 'gridrec' - init_recon: null -- method: rescale_to_int - module_path: httomolibgpu.misc.rescale - parameters: - perc_range_min: 0.0 - perc_range_max: 100.0 - bits: 8 -- method: save_to_images - module_path: httomolib.misc.images - parameters: - subfolder_name: images - axis: auto - file_format: tif - jpeg_quality: 95 diff --git a/tests/samples/pipeline_template_examples/pipeline_cpu5.yaml b/tests/samples/pipeline_template_examples/pipeline_cpu5.yaml deleted file mode 100644 index e0e1edbe7..000000000 --- a/tests/samples/pipeline_template_examples/pipeline_cpu5.yaml +++ /dev/null @@ -1,34 +0,0 @@ -- method: standard_tomo - module_path: httomo.data.hdf.loaders - parameters: - data_path: entry1/tomo_entry/data/data - image_key_path: entry1/tomo_entry/instrument/detector/image_key - rotation_angles: - data_path: /entry1/tomo_entry/data/rotation_angle - preview: - detector_x: - start: mid - start_offset : -50 - stop: mid - stop_offset : 50 - detector_y: - start: begin - start_offset : 32 - stop: end - stop_offset : -32 -- method: normalize - module_path: tomopy.prep.normalize - parameters: - cutoff: null - averaging: mean -- method: minus_log - module_path: tomopy.prep.normalize - parameters: {} -- method: recon - module_path: tomopy.recon.algorithm - parameters: - center: 50.5 - sinogram_order: false - algorithm: 'gridrec' - init_recon: null - save_result: false \ No newline at end of file diff --git a/tests/samples/pipeline_template_examples/pipeline_gpu1.yaml b/tests/samples/pipeline_template_examples/pipeline_gpu1.yaml deleted file mode 100644 index ad652df5f..000000000 --- a/tests/samples/pipeline_template_examples/pipeline_gpu1.yaml +++ /dev/null @@ -1,57 +0,0 @@ -- method: standard_tomo - module_path: httomo.data.hdf.loaders - parameters: - data_path: entry1/tomo_entry/data/data - image_key_path: entry1/tomo_entry/instrument/detector/image_key - rotation_angles: - data_path: /entry1/tomo_entry/data/rotation_angle -- method: find_center_vo - module_path: httomolibgpu.recon.rotation - parameters: - ind: mid - smin: -50 - smax: 50 - srad: 6.0 - step: 0.25 - ratio: 0.5 - drop: 20 - id: centering - side_outputs: - cor: centre_of_rotation -- method: remove_outlier - module_path: httomolibgpu.misc.corr - parameters: - dif: 0.1 - kernel_size: 3 -- method: normalize - module_path: httomolibgpu.prep.normalize - parameters: - cutoff: 10.0 - minus_log: true - nonnegativity: false - remove_nans: false -- method: remove_stripe_based_sorting - module_path: httomolibgpu.prep.stripe - parameters: - size: 11 - dim: 1 -- method: FBP - module_path: httomolibgpu.recon.algorithm - parameters: - center: ${{centering.side_outputs.centre_of_rotation}} - filter_freq_cutoff: 0.6 - recon_size: null - recon_mask_radius: null - save_result: true -- method: rescale_to_int - module_path: httomolibgpu.misc.rescale - parameters: - perc_range_min: 0.0 - perc_range_max: 100.0 - bits: 8 -- method: save_to_images - module_path: httomolib.misc.images - parameters: - subfolder_name: images - axis: auto - file_format: tif diff --git a/tests/samples/pipeline_template_examples/testing/testing_pipeline.yaml b/tests/samples/pipeline_template_examples/testing/testing_pipeline.yaml deleted file mode 100644 index 0916805e4..000000000 --- a/tests/samples/pipeline_template_examples/testing/testing_pipeline.yaml +++ /dev/null @@ -1,49 +0,0 @@ -- method: normalize - module_path: tomopy.prep.normalize - parameters: - cutoff: null - averaging: mean -- method: minus_log - module_path: tomopy.prep.normalize - parameters: {} -- method: find_center_vo - module_path: tomopy.recon.rotation - parameters: - ind: mid - smin: -50 - smax: 50 - srad: 6 - step: 0.25 - ratio: 0.5 - drop: 20 - id: centering - side_outputs: - cor: centre_of_rotation -- method: remove_stripe_fw - module_path: tomopy.prep.stripe - parameters: - level: null - wname: db5 - sigma: 2 - pad: true -- method: recon - module_path: tomopy.recon.algorithm - parameters: - center: ${{centering.side_outputs.centre_of_rotation}} - sinogram_order: false - algorithm: gridrec - init_recon: null - save_result: true -- method: rescale_to_int - module_path: httomolibgpu.misc.rescale - parameters: - perc_range_min: 0.0 - perc_range_max: 100.0 - bits: 8 -- method: save_to_images - module_path: httomolib.misc.images - parameters: - subfolder_name: images - axis: 1 - file_format: tif - jpeg_quality: 95 diff --git a/tests/scripts/create_numpy_from_hdf5.py b/tests/scripts/create_numpy_from_hdf5.py new file mode 100644 index 000000000..817ff0041 --- /dev/null +++ b/tests/scripts/create_numpy_from_hdf5.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------- +# Copyright 2022 Diamond Light Source Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +# Created By : Tomography Team +# Created Date: 22/January/2025 +# version ='0.1' +# --------------------------------------------------------------------------- +"""Script that generates YAML pipeline for HTTomo using YAML templates from httomo-backends +(should be already installed in your environment). + +Please run the generator as: + python -m create_numpy_from_hdf5 -i /path/to/file.hdf5 -o /path/to/output/file.npz +""" +import argparse +import os +import h5py +import numpy as np + + +def create_numpy_from_hdf5(path_to_hdf5: str, path_to_output_file: str) -> int: + """ + Args: + path_to_hdf5: A path to the hdf5 file from which data needs to be extracted. + path_to_output_file: Output path to the saved dataset as numpy array. + + Returns: + returns zero if the extraction of data is successful + """ + h5f = h5py.File(path_to_hdf5, "r") + path_to_data = "data/" + slice_axis0 = h5f[path_to_data][0, :, :] + slice_axis1 = h5f[path_to_data][:, 0, :] + size_axisY, size_axisZ = np.shape(slice_axis0) + size_axisX, _ = np.shape(slice_axis1) + + slices = 10 + # we will be iterating over axis Y + data_selection = np.zeros((slices, size_axisX, size_axisZ), dtype=np.float32) + step = size_axisY // (slices + 2) + index_prog = step + for i in range(slices): + data_selection[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + + np.savez(path_to_output_file, data=data_selection, axis_slice=size_axisY) + + return 0 + + +def get_args(): + parser = argparse.ArgumentParser( + description="Script that creates a numpy array from hdf5" + "reconstruction file and saves it on disk." + ) + parser.add_argument( + "-i", + "--input", + type=str, + default=None, + help="A path to the hdf5 file.", + ) + parser.add_argument( + "-o", + "--output", + type=str, + default=None, + help="Output path to the saved dataset as numpy array.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = get_args() + path_to_hdf5 = args.input + path_to_output_file = args.output + return_val = create_numpy_from_hdf5(path_to_hdf5, path_to_output_file) + if return_val == 0: + message_str = ( + f"Numpy file {path_to_output_file} has been created from {path_to_hdf5}." + ) + print(message_str) diff --git a/tests/test_data/360scan/360scan.hdf b/tests/test_data/360scan/360scan.hdf deleted file mode 100644 index 7368e1c91..000000000 Binary files a/tests/test_data/360scan/360scan.hdf and /dev/null differ diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py deleted file mode 100644 index 71ab3e1d6..000000000 --- a/tests/test_pipeline.py +++ /dev/null @@ -1,631 +0,0 @@ -import re -import subprocess -from typing import Callable, List, Tuple - -import h5py -import numpy as np -import pytest -from numpy.testing import assert_allclose -from PIL import Image -from plumbum import local - -PATTERN = re.compile(r"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]") - - -def _get_log_contents(file): - """return contents of the user.log file""" - with open(file, "r") as f: - log_contents = f.read() - - #: check that the generated log file has no ansi escape sequence - # assert not PATTERN.search(log_contents) - - return log_contents - - -def _compare_two_yamls(original_yaml, copied_yaml): - with open(original_yaml, "r") as oy, open(copied_yaml, "r") as cy: - return oy.read() == cy.read() - - -def _check_yaml(files: List, input_yaml: str): - # check that the contents of the copied YAML in the output directory matches - # the contents of the input YAML - copied_yaml_path = list(filter(lambda x: ".yaml" in x, files)).pop() - assert _compare_two_yamls(input_yaml, copied_yaml_path) - - -def _check_tif(files: List, number: int, shape: Tuple): - # check the .tif files - tif_files = list(filter(lambda x: ".tif" in x, files)) - assert len(tif_files) == number - - # check that the image size is correct - imarray = np.array(Image.open(tif_files[0])) - assert imarray.shape == shape - - -def test_tomo_standard_testing_pipeline_output( - get_files: Callable, - cmd, - standard_data, - standard_loader, - testing_pipeline, - output_folder, - merge_yamls, -): - cmd.pop(4) #: don't save all - cmd.insert(6, standard_data) - merge_yamls(standard_loader, testing_pipeline) - cmd.insert(7, "temp.yaml") - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 7 - - _check_yaml(files, "temp.yaml") - _check_tif(files, 3, (160, 160)) - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - for file_to_open in h5_files: - if "tomopy-recon-tomo-gridrec.h5" in file_to_open: - with h5py.File(file_to_open, "r") as f: - assert f["data"].shape == (160, 3, 160) - assert f["data"].dtype == np.float32 - assert_allclose(np.mean(f["data"]), 0.0015362317, atol=1e-6, rtol=1e-6) - assert_allclose(np.sum(f["data"]), 117.9826, atol=1e-6, rtol=1e-6) - - #: some basic testing of the generated user.log file, because running the whole pipeline again - #: will slow down the execution of the test suite. - #: It will be worth moving the unit tests for the logger to a separate file - #: once we generate different log files for each MPI process and we can compare them. - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - user_log_file = list(filter(lambda x: "user.log" in x, files)) - assert len(verbose_log_file) == 1 - assert len(user_log_file) == 1 - - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert f"{user_log_file[0]}" in verbose_log_contents - assert "The full dataset shape is (220, 128, 160)" in verbose_log_contents - assert "Loading data: tests/test_data/tomo_standard.nxs" in verbose_log_contents - assert "Path to data: entry1/tomo_entry/data/data" in verbose_log_contents - assert "Preview: (0:180, 57:60, 0:160)" in verbose_log_contents - assert "Data shape is (180, 3, 160) of type uint16" in verbose_log_contents - - -def test_run_pipeline_cpu1_yaml( - get_files: Callable, cmd, standard_data, yaml_cpu_pipeline1, output_folder -): - cmd.pop(4) #: don't save all - cmd.insert(6, standard_data) - cmd.insert(7, yaml_cpu_pipeline1) - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 132 # 128 images + yaml, 2 logfiles, intermdiate - - _check_tif(files, 128, (160, 160)) - - log_files = list(filter(lambda x: ".log" in x, files)) - assert len(log_files) == 2 - - concise_log_file = list(filter(lambda x: "user.log" in x, files)) - concise_log_contents = _get_log_contents(concise_log_file[0]) - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert f"{concise_log_file[0]}" in concise_log_contents - assert "The center of rotation is 79.5" in concise_log_contents - assert "The full dataset shape is (220, 128, 160)" in verbose_log_contents - assert "Loading data: tests/test_data/tomo_standard.nxs" in verbose_log_contents - assert "Path to data: entry1/tomo_entry/data/data" in verbose_log_contents - assert "Preview: (0:180, 0:128, 0:160)" in verbose_log_contents - assert "Data shape is (180, 128, 160) of type uint16" in verbose_log_contents - - -def test_run_pipeline_cpu2_yaml( - get_files: Callable, cmd, standard_data, yaml_cpu_pipeline2, output_folder -): - cmd.pop(4) #: don't save all - cmd.insert(6, standard_data) - cmd.insert(7, yaml_cpu_pipeline2) - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 34 - - _check_tif(files, 30, (160, 160)) - - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - user_log_file = list(filter(lambda x: "user.log" in x, files)) - assert len(verbose_log_file) == 1 - assert len(user_log_file) == 1 - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - for file_to_open in h5_files: - if "tomopy-recon-tomo-gridrec.h5" in file_to_open: - with h5py.File(file_to_open, "r") as f: - assert f["data"].shape == (160, 30, 160) - assert f["data"].dtype == np.float32 - assert_allclose(np.sum(f["data"]), 694.70306, atol=1e-6, rtol=1e-6) - - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert f"{user_log_file[0]}" in verbose_log_contents - assert "The full dataset shape is (220, 128, 160)" in verbose_log_contents - assert "Loading data: tests/test_data/tomo_standard.nxs" in verbose_log_contents - assert "Path to data: entry1/tomo_entry/data/data" in verbose_log_contents - assert "Preview: (0:180, 30:60, 0:160)" in verbose_log_contents - assert "Data shape is (180, 30, 160) of type uint16" in verbose_log_contents - - -def test_run_pipeline_cpu3_yaml( - get_files: Callable, cmd, standard_data, yaml_cpu_pipeline3, output_folder -): - cmd.pop(4) #: don't save all - cmd.insert(6, standard_data) - cmd.insert(7, yaml_cpu_pipeline3) - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 132 # 128 images + yaml, 2 logfiles, intermdiate - - _check_tif(files, 128, (160, 160)) - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - user_log_file = list(filter(lambda x: "user.log" in x, files)) - assert len(verbose_log_file) == 1 - assert len(user_log_file) == 1 - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert f"{user_log_file[0]}" in verbose_log_contents - assert "The full dataset shape is (220, 128, 160)" in verbose_log_contents - assert "Loading data: tests/test_data/tomo_standard.nxs" in verbose_log_contents - assert "Path to data: entry1/tomo_entry/data/data" in verbose_log_contents - assert "Preview: (0:180, 0:128, 0:160)" in verbose_log_contents - assert "Data shape is (180, 128, 160) of type uint16" in verbose_log_contents - assert " Global min -0.014979" in verbose_log_contents - assert " Global max 0.04177" in verbose_log_contents - assert " Global mean 0.0016174" in verbose_log_contents - - -def test_run_pipeline_cpu4_yaml( - get_files: Callable, cmd, standard_data, yaml_cpu_pipeline4, output_folder -): - cmd.pop(4) #: don't save all - cmd.insert(6, standard_data) - cmd.insert(7, yaml_cpu_pipeline4) - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 132 # 128 images + yaml, 2 logfiles, intermdiate - - _check_tif(files, 128, (160, 160)) - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - log_files = list(filter(lambda x: ".log" in x, files)) - assert len(log_files) == 2 - concise_log_file = list(filter(lambda x: "user.log" in x, files)) - concise_log_contents = _get_log_contents(concise_log_file[0]) - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert f"{concise_log_file[0]}" in concise_log_contents - assert "The center of rotation is 79.5" in concise_log_contents - assert "The full dataset shape is (220, 128, 160)" in verbose_log_contents - assert "Loading data: tests/test_data/tomo_standard.nxs" in verbose_log_contents - assert "Path to data: entry1/tomo_entry/data/data" in verbose_log_contents - assert "Preview: (0:180, 0:128, 0:160)" in verbose_log_contents - assert "Data shape is (180, 128, 160) of type uint16" in verbose_log_contents - - -def test_run_pipeline_cpu5_yaml( - get_files: Callable, cmd, standard_data, yaml_cpu_pipeline5, output_folder -): - cmd.pop(4) #: don't save all - cmd.insert(6, standard_data) - cmd.insert(7, yaml_cpu_pipeline5) - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 3 - - # explore the debug log and the previewed data - log_files = list(filter(lambda x: ".log" in x, files)) - assert len(log_files) == 2 - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert "Preview: (0:180, 32:96, 29:129)" in verbose_log_contents - assert "Data shape is (180, 64, 100) of type uint16" in verbose_log_contents - - -@pytest.mark.cupy -def test_run_pipeline_gpu1_yaml( - get_files: Callable, cmd, standard_data, yaml_gpu_pipeline1, output_folder -): - cmd.pop(4) #: don't save all - cmd.insert(6, standard_data) - cmd.insert(7, yaml_gpu_pipeline1) - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 132 - - _check_tif(files, 128, (160, 160)) - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - for file_to_open in h5_files: - if "httomolibgpu-FBP-tomo.h5" in file_to_open: - with h5py.File(file_to_open, "r") as f: - assert f["data"].shape == (160, 128, 160) - assert f["data"].dtype == np.float32 - assert_allclose(np.sum(f["data"]), 2615.7332, atol=1e-6, rtol=1e-6) - - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - user_log_file = list(filter(lambda x: "user.log" in x, files)) - assert len(verbose_log_file) == 1 - assert len(user_log_file) == 1 - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert f"{user_log_file[0]}" in verbose_log_contents - assert "The full dataset shape is (220, 128, 160)" in verbose_log_contents - assert "Loading data: tests/test_data/tomo_standard.nxs" in verbose_log_contents - assert "Path to data: entry1/tomo_entry/data/data" in verbose_log_contents - assert "Preview: (0:180, 0:128, 0:160)" in verbose_log_contents - assert "Data shape is (180, 128, 160) of type uint16" in verbose_log_contents - assert "The amount of the available GPU memory is" in verbose_log_contents - assert ( - "Using GPU 0 to transfer data of shape (180, 128, 160)" in verbose_log_contents - ) - - -def test_tomo_standard_testing_pipeline_output_with_save_all( - get_files: Callable, - cmd, - standard_data, - standard_loader, - testing_pipeline, - output_folder, - merge_yamls, -): - cmd.insert(7, standard_data) - merge_yamls(standard_loader, testing_pipeline) - cmd.insert(8, "temp.yaml") - cmd.insert(9, output_folder) - subprocess.check_output(cmd) - - files = get_files("output_dir/") - assert len(files) == 11 - - _check_yaml(files, "temp.yaml") - _check_tif(files, 3, (160, 160)) - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 5 - - for file_to_open in h5_files: - if "tomopy-recon-tomo-gridrec.h5" in file_to_open: - with h5py.File(file_to_open, "r") as f: - assert f["data"].shape == (160, 3, 160) - assert f["data"].dtype == np.float32 - assert_allclose(np.mean(f["data"]), 0.0015362317, atol=1e-6, rtol=1e-6) - assert_allclose(np.sum(f["data"]), 117.9826, atol=1e-6, rtol=1e-6) - - -def test_i12_testing_pipeline_output( - get_files: Callable, - cmd, - i12_data, - i12_loader, - testing_pipeline, - output_folder, - merge_yamls, -): - cmd.insert(7, i12_data) - merge_yamls(i12_loader, testing_pipeline) - cmd.insert(8, "temp.yaml") - cmd.insert(9, output_folder) - subprocess.check_output(cmd) - - files = get_files("output_dir/") - assert len(files) == 18 - - _check_yaml(files, "temp.yaml") - - log_files = list(filter(lambda x: ".log" in x, files)) - assert len(log_files) == 2 - - tif_files = list(filter(lambda x: ".tif" in x, files)) - assert len(tif_files) == 10 - - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 5 - - gridrec_recon = list(filter(lambda x: "recon-gridrec.h5" in x, h5_files))[0] - minus_log_tomo = list(filter(lambda x: "minus_log.h5" in x, h5_files))[0] - remove_stripe_fw_tomo = list( - filter(lambda x: "remove_stripe_fw.h5" in x, h5_files) - )[0] - normalize_tomo = list(filter(lambda x: "normalize.h5" in x, h5_files))[0] - - with h5py.File(gridrec_recon, "r") as f: - assert f["data"].shape == (192, 10, 192) - assert_allclose(np.sum(f["data"]), 2157.03, atol=1e-2, rtol=1e-6) - assert_allclose(np.mean(f["data"]), 0.0058513316, atol=1e-6, rtol=1e-6) - with h5py.File(minus_log_tomo, "r") as f: - assert_allclose(np.sum(f["data"]), 1756628.4, atol=1e-6, rtol=1e-6) - assert_allclose(np.mean(f["data"]), 1.2636887, atol=1e-6, rtol=1e-6) - with h5py.File(remove_stripe_fw_tomo, "r") as f: - assert_allclose(np.sum(f["data"]), 1766357.8, atol=1e-6, rtol=1e-6) - assert_allclose(np.mean(f["data"]), 1.2706878, atol=1e-6, rtol=1e-6) - with h5py.File(normalize_tomo, "r") as f: - assert f["data"].shape == (724, 10, 192) - assert_allclose(np.sum(f["data"]), 393510.72, atol=1e-6, rtol=1e-6) - assert_allclose(np.mean(f["data"]), 0.28308493, atol=1e-6, rtol=1e-6) - - concise_log_file = list(filter(lambda x: "user.log" in x, files)) - concise_log_contents = _get_log_contents(concise_log_file[0]) - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert "The center of rotation is 95.5" in concise_log_contents - assert "The full dataset shape is (724, 10, 192)" in verbose_log_contents - assert ( - "Loading data: tests/test_data/i12/separate_flats_darks/i12_dynamic_start_stop180.nxs" - in verbose_log_contents - ) - assert "Path to data: /1-TempPlugin-tomo/data" in verbose_log_contents - assert "Preview: (0:724, 0:10, 0:192)" in verbose_log_contents - - -# TODO: Add back in when ignoring darks/flats is added to the new loader -# -# def test_i12_testing_ignore_darks_flats_pipeline_output( -# get_files: Callable, -# cmd, -# i12_data, -# i12_loader_ignore_darks_flats, -# testing_pipeline, -# output_folder, -# merge_yamls, -# ): -# cmd.insert(7, i12_data) -# merge_yamls(i12_loader_ignore_darks_flats, testing_pipeline) -# cmd.insert(8, "temp.yaml") -# cmd.insert(9, output_folder) -# subprocess.check_output(cmd) -# -# files = get_files("output_dir/") -# assert len(files) == 16 -# -# _check_yaml(files, "temp.yaml") -# -# log_files = list(filter(lambda x: ".log" in x, files)) -# assert len(log_files) == 1 -# -# tif_files = list(filter(lambda x: ".tif" in x, files)) -# assert len(tif_files) == 10 -# -# h5_files = list(filter(lambda x: ".h5" in x, files)) -# assert len(h5_files) == 4 -# -# log_contents = _get_log_contents(log_files[0]) -# assert "The full dataset shape is (724, 10, 192)" in log_contents -# assert ( -# "Loading data: tests/test_data/i12/separate_flats_darks/i12_dynamic_start_stop180.nxs" -# in log_contents -# ) -# assert "Path to data: /1-TempPlugin-tomo/data" in log_contents -# assert "Preview: (0:724, 0:10, 0:192)" in log_contents -# assert "Running save_task_1 (pattern=projection): save_intermediate_data..." in log_contents -# assert "Running save_task_2 (pattern=projection): save_intermediate_data..." in log_contents -# assert "Running save_task_4 (pattern=sinogram): save_intermediate_data..." in log_contents -# assert "The center of rotation for sinogram is 95.5" in log_contents -# assert "Running save_task_5 (pattern=sinogram): save_intermediate_data..." in log_contents - - -def test_diad_testing_pipeline_output( - get_files: Callable, - cmd, - diad_data, - diad_loader, - testing_pipeline, - output_folder, - merge_yamls, -): - cmd.insert(7, diad_data) - merge_yamls(diad_loader, testing_pipeline) - cmd.insert(8, "temp.yaml") - cmd.insert(9, output_folder) - subprocess.check_output(cmd) - - files = get_files("output_dir/") - assert len(files) == 10 - - _check_yaml(files, "temp.yaml") - _check_tif(files, 2, (26, 26)) - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 5 - - for file_to_open in h5_files: - if "tomopy-normalize-tomo.h5" in file_to_open: - with h5py.File(file_to_open, "r") as f: - assert f["data"].shape == (3001, 2, 26) - assert f["data"].dtype == np.float32 - assert_allclose(np.mean(f["data"]), 0.847944, atol=1e-6, rtol=1e-6) - assert_allclose(np.sum(f["data"]), 132323.36, atol=1e-6, rtol=1e-6) - if "tomopy-recon-tomo-gridrec.h5" in file_to_open: - with h5py.File(file_to_open, "r") as f: - assert f["data"].shape == (26, 2, 26) - assert_allclose(np.mean(f["data"]), 0.005883, atol=1e-6, rtol=1e-6) - assert_allclose(np.sum(f["data"]), 7.954298, atol=1e-6, rtol=1e-6) - - log_files = list(filter(lambda x: ".log" in x, files)) - assert len(log_files) == 2 - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert "The full dataset shape is (3201, 22, 26)" in verbose_log_contents - assert ( - "Loading data: tests/test_data/k11_diad/k11-18014.nxs" in verbose_log_contents - ) - assert "Path to data: /entry/imaging/data" in verbose_log_contents - assert "Preview: (100:3101, 5:7, 0:26)" in verbose_log_contents - assert "Data shape is (3001, 2, 26) of type uint16" in verbose_log_contents - - -@pytest.mark.cupy -def test_run_diad_pipeline_gpu( - get_files: Callable, cmd, diad_data, diad_pipeline_gpu, output_folder -): - cmd.pop(4) #: don't save all - cmd.insert(6, diad_data) - cmd.insert(7, diad_pipeline_gpu) - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 11 - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - log_files = list(filter(lambda x: ".log" in x, files)) - assert len(log_files) == 2 - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert "The full dataset shape is (3201, 22, 26)" in verbose_log_contents - assert ( - "Loading data: tests/test_data/k11_diad/k11-18014.nxs" in verbose_log_contents - ) - assert "Path to data: /entry/imaging/data" in verbose_log_contents - assert "Preview: (100:3101, 8:15, 0:26)" in verbose_log_contents - assert "Data shape is (3001, 7, 26) of type uint16" in verbose_log_contents - - -@pytest.mark.cupy -def test_run_pipeline_360deg_gpu2( - get_files: Callable, cmd, data360, yaml_gpu_pipeline360_2, output_folder -): - cmd.pop(4) #: don't save all - cmd.insert(6, data360) - cmd.insert(7, yaml_gpu_pipeline360_2) - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 7 - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - log_files = list(filter(lambda x: ".log" in x, files)) - assert len(log_files) == 2 - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert "The full dataset shape is (3751, 3, 2560)" in verbose_log_contents - assert "Loading data: tests/test_data/360scan/360scan.hdf" in verbose_log_contents - assert "Path to data: entry1/tomo_entry/data/data" in verbose_log_contents - assert "Data shape is (3601, 3, 2560) of type uint16" in verbose_log_contents - - -@pytest.mark.cupy -def test_run_gpu_pipeline_sweep_cor( - get_files: Callable, cmd, standard_data, yaml_gpu_pipeline_sweep_cor, output_folder -): - cmd.pop(4) #: don't save all - cmd.insert(6, standard_data) - cmd.insert(7, yaml_gpu_pipeline_sweep_cor) - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 9 - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 0 - - log_files = list(filter(lambda x: ".log" in x, files)) - assert len(log_files) == 2 - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert "Data shape is (180, 7, 160) of type uint16" in verbose_log_contents - assert "Total number of values across all processes: 6" in verbose_log_contents - assert "Values executed in this process: 6" in verbose_log_contents - - -@pytest.mark.cupy -def test_run_gpu_pipeline_sweep_paganin( - get_files: Callable, - cmd, - standard_data, - yaml_gpu_pipeline_sweep_paganin, - output_folder, -): - cmd.pop(4) #: don't save all - cmd.insert(6, standard_data) - cmd.insert(7, yaml_gpu_pipeline_sweep_paganin) - cmd.insert(8, output_folder) - subprocess.check_output(cmd) - - # recurse through output_dir and check that all files are there - files = get_files("output_dir/") - assert len(files) == 104 - - #: check the generated h5 files - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - log_files = list(filter(lambda x: ".log" in x, files)) - assert len(log_files) == 2 - verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) - verbose_log_contents = _get_log_contents(verbose_log_file[0]) - - assert "Data shape is (180, 7, 160) of type uint16" in verbose_log_contents - assert "Total number of values across all processes: 50" in verbose_log_contents - assert "Values executed in this process: 50" in verbose_log_contents - assert "Parameter name: alpha" in verbose_log_contents diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py new file mode 100644 index 000000000..fbb4d6200 --- /dev/null +++ b/tests/test_pipeline_big.py @@ -0,0 +1,399 @@ +import re +import subprocess +from typing import Callable, List, Tuple, Union + +import h5py +import numpy as np +import pytest +from numpy.testing import assert_allclose +from plumbum import local +from .conftest import change_value_parameters_method_pipeline + +# NOTE: those tests have path integrated that are compatible with running jobs in Jenkins at DLS infrastructure. + + +@pytest.mark.full_data +def test_pipeline_gpu_FBP_diad_k11_38731_in_disk( + get_files: Callable, + cmd, + diad_k11_38731, + gpu_pipeline_diad_FBP_noimagesaving, + gpu_diad_FBP_k11_38731_npz, + output_folder, +): + # NOTE that the intermediate file with file-based processing will be saved to /tmp + + cmd.pop(4) #: don't save all + cmd.insert(5, diad_k11_38731) + cmd.insert(7, gpu_pipeline_diad_FBP_noimagesaving) + cmd.insert(8, output_folder) + cmd.insert(9, "--max-memory") + cmd.insert(10, "40G") + cmd.insert(11, "--reslice-dir") + cmd.insert(12, "/scratch/jenkins_agent/workspace/") + + subprocess.check_output(cmd) + + files = get_files("output_dir/") + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = gpu_diad_FBP_k11_38731_npz["data"] + axis_slice = gpu_diad_FBP_k11_38731_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "httomolibgpu-FBP" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-6 + + +@pytest.mark.full_data +def test_pipeline_gpu_FBP_diad_k11_38731_in_memory( + get_files: Callable, + cmd, + diad_k11_38731, + gpu_pipeline_diad_FBP_noimagesaving, + gpu_diad_FBP_k11_38731_npz, + output_folder, +): + cmd.pop(4) #: don't save all + cmd.insert(5, diad_k11_38731) + cmd.insert(7, gpu_pipeline_diad_FBP_noimagesaving) + cmd.insert(8, output_folder) + + subprocess.check_output(cmd) + + files = get_files("output_dir/") + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = gpu_diad_FBP_k11_38731_npz["data"] + axis_slice = gpu_diad_FBP_k11_38731_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "httomolibgpu-FBP" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-6 + + +@pytest.mark.full_data +def test_pipeline_gpu_FBP_diad_k11_38730_in_disk( + get_files: Callable, + cmd, + diad_k11_38730, + gpu_pipeline_diad_FBP_noimagesaving, + gpu_diad_FBP_k11_38730_npz, + output_folder, +): + # NOTE that the intermediate file with file-based processing will be saved to /tmp + + cmd.pop(4) #: don't save all + cmd.insert(5, diad_k11_38730) + cmd.insert(7, gpu_pipeline_diad_FBP_noimagesaving) + cmd.insert(8, output_folder) + cmd.insert(9, "--max-memory") + cmd.insert(10, "40G") + cmd.insert(11, "--reslice-dir") + cmd.insert(12, "/scratch/jenkins_agent/workspace/") + + subprocess.check_output(cmd) + + files = get_files("output_dir/") + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = gpu_diad_FBP_k11_38730_npz["data"] + axis_slice = gpu_diad_FBP_k11_38730_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "httomolibgpu-FBP" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-6 + + +@pytest.mark.full_data +def test_pipeline_gpu_FBP_diad_k11_38730_in_memory( + get_files: Callable, + cmd, + diad_k11_38730, + gpu_pipeline_diad_FBP_noimagesaving, + gpu_diad_FBP_k11_38730_npz, + output_folder, +): + cmd.pop(4) #: don't save all + cmd.insert(5, diad_k11_38730) + cmd.insert(7, gpu_pipeline_diad_FBP_noimagesaving) + cmd.insert(8, output_folder) + + subprocess.check_output(cmd) + + files = get_files("output_dir/") + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = gpu_diad_FBP_k11_38730_npz["data"] + axis_slice = gpu_diad_FBP_k11_38730_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "httomolibgpu-FBP" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-6 + + +@pytest.mark.full_data +def test_pipeline_gpu_FBP_denoising_i13_177906_preview( + get_files: Callable, + cmd, + i13_177906, + gpu_pipelineFBP_denoising, + gpu_FBP_TVdenoising_i13_177906_npz, + output_folder, +): + + change_value_parameters_method_pipeline( + gpu_pipelineFBP_denoising, + method=[ + "standard_tomo", + ], + key=[ + "preview", + ], + value=[ + {"detector_y": {"start": 900, "stop": 1200}}, + ], + ) + + # do not save the result of FBP + change_value_parameters_method_pipeline( + gpu_pipelineFBP_denoising, + method=[ + "FBP", + ], + key=[ + "recon_size", + ], + value=[ + None, + ], + save_result=False, + ) + + # save the result of denoising instead + change_value_parameters_method_pipeline( + gpu_pipelineFBP_denoising, + method=[ + "total_variation_PD", + "total_variation_PD", + ], + key=[ + "regularisation_parameter", + "iterations", + ], + value=[ + 1.0e-04, + 25, + ], + save_result=True, + ) + + cmd.pop(4) #: don't save all + cmd.insert(5, i13_177906) + cmd.insert(7, gpu_pipelineFBP_denoising) + cmd.insert(8, output_folder) + + subprocess.check_output(cmd) + + files = get_files("output_dir/") + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = gpu_FBP_TVdenoising_i13_177906_npz["data"] + axis_slice = gpu_FBP_TVdenoising_i13_177906_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "total_variation_PD" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-6 + + +@pytest.mark.full_data +def test_pipeline_gpu_360_paganin_FBP_i13_179623_preview( + get_files: Callable, + cmd, + i13_179623, + gpu_pipeline_360_paganin_FBP, + gpu_FBP_paganin_i13_179623_npz, + output_folder, +): + change_value_parameters_method_pipeline( + gpu_pipeline_360_paganin_FBP, + method=[ + "standard_tomo", + "normalize", + "find_center_360", + "sino_360_to_180", + "paganin_filter_tomopy", + "paganin_filter_tomopy", + ], + key=[ + "preview", + "minus_log", + "ind", + "rotation", + "energy", + "alpha", + ], + value=[ + {"detector_y": {"start": 900, "stop": 1200}}, + False, + "mid", + "right", + 15.0, + 0.1, + ], + ) + + cmd.pop(4) #: don't save all + cmd.insert(5, i13_179623) + cmd.insert(7, gpu_pipeline_360_paganin_FBP) + cmd.insert(8, output_folder) + + subprocess.check_output(cmd) + + files = get_files("output_dir/") + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = gpu_FBP_paganin_i13_179623_npz["data"] + axis_slice = gpu_FBP_paganin_i13_179623_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "httomolibgpu-FBP" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-6 diff --git a/tests/test_pipeline_small.py b/tests/test_pipeline_small.py new file mode 100644 index 000000000..a65e698ed --- /dev/null +++ b/tests/test_pipeline_small.py @@ -0,0 +1,412 @@ +import re +import subprocess +from typing import Callable, List, Tuple, Union + +import h5py +import numpy as np +import pytest +from numpy.testing import assert_allclose +from PIL import Image +from plumbum import local +from .conftest import change_value_parameters_method_pipeline + +PATTERN = re.compile(r"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]") + + +def _get_log_contents(file): + """return contents of the user.log file""" + with open(file, "r") as f: + log_contents = f.read() + + #: check that the generated log file has no ansi escape sequence + # assert not PATTERN.search(log_contents) + + return log_contents + + +def _compare_two_yamls(original_yaml, copied_yaml): + with open(original_yaml, "r") as oy, open(copied_yaml, "r") as cy: + return oy.read() == cy.read() + + +def _check_yaml(files: List, input_yaml: str): + # check that the contents of the copied YAML in the output directory matches + # the contents of the input YAML + copied_yaml_path = list(filter(lambda x: ".yaml" in x, files)).pop() + assert _compare_two_yamls(input_yaml, copied_yaml_path) + + +def _check_tif(files: List, number: int, shape: Tuple): + # check the .tif files + tif_files = list(filter(lambda x: ".tif" in x, files)) + assert len(tif_files) == number + + # check that the image size is correct + imarray = np.array(Image.open(tif_files[0])) + assert imarray.shape == shape + + +@pytest.mark.small_data +def test_run_pipeline_cpu_gridrec( + get_files: Callable, cmd, standard_data, cpu_pipeline_gridrec, output_folder +): + cmd.pop(4) #: don't save all + cmd.insert(6, standard_data) + cmd.insert(7, cpu_pipeline_gridrec) + cmd.insert(8, output_folder) + + subprocess.check_output(cmd) + + # recurse through output_dir and check that all files are there + files = get_files("output_dir/") + assert len(files) == 132 # 128 images + yaml, 2 logfiles, intermediate + + _check_tif(files, 128, (160, 160)) + + log_files = list(filter(lambda x: ".log" in x, files)) + assert len(log_files) == 2 + + concise_log_file = list(filter(lambda x: "user.log" in x, files)) + concise_log_contents = _get_log_contents(concise_log_file[0]) + verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) + verbose_log_contents = _get_log_contents(verbose_log_file[0]) + + assert f"{concise_log_file[0]}" in concise_log_contents + assert "The center of rotation is 79.5" in concise_log_contents + assert "The full dataset shape is (220, 128, 160)" in verbose_log_contents + assert "Loading data: tests/test_data/tomo_standard.nxs" in verbose_log_contents + assert "Path to data: /entry1/tomo_entry/data/data" in verbose_log_contents + assert "Preview: (0:180, 0:128, 0:160)" in verbose_log_contents + assert "Data shape is (180, 128, 160) of type uint16" in verbose_log_contents + + +@pytest.mark.small_data +def test_run_pipeline_gpu_FBP( + get_files: Callable, cmd, standard_data, gpu_pipelineFBP, output_folder +): + cmd.pop(4) #: don't save all + cmd.insert(6, standard_data) + cmd.insert(7, gpu_pipelineFBP) + cmd.insert(8, output_folder) + subprocess.check_output(cmd) + + # recurse through output_dir and check that all files are there + files = get_files("output_dir/") + assert len(files) == 132 + + _check_tif(files, 128, (160, 160)) + + #: check the generated h5 files + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + for file_to_open in h5_files: + if "httomolibgpu-FBP" in file_to_open: + h5f = h5py.File(file_to_open, "r") + assert h5f["data"].shape == (160, 128, 160) + assert h5f["data"].dtype == np.float32 + h5f.close() + else: + raise FileNotFoundError("File with httomolibgpu-FBP string cannot be found") + + verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) + user_log_file = list(filter(lambda x: "user.log" in x, files)) + assert len(verbose_log_file) == 1 + assert len(user_log_file) == 1 + verbose_log_contents = _get_log_contents(verbose_log_file[0]) + + assert f"{user_log_file[0]}" in verbose_log_contents + assert "The full dataset shape is (220, 128, 160)" in verbose_log_contents + assert "Loading data: tests/test_data/tomo_standard.nxs" in verbose_log_contents + assert "Path to data: /entry1/tomo_entry/data/data" in verbose_log_contents + assert "Preview: (0:180, 0:128, 0:160)" in verbose_log_contents + assert "Data shape is (180, 128, 160) of type uint16" in verbose_log_contents + assert "The amount of the available GPU memory is" in verbose_log_contents + assert ( + "Using GPU 0 to transfer data of shape (180, 128, 160)" in verbose_log_contents + ) + + +@pytest.mark.small_data +def test_run_pipeline_gpu_denoise( + get_files: Callable, + cmd, + standard_data, + gpu_pipelineFBP_denoising, + output_folder, +): + change_value_parameters_method_pipeline( + gpu_pipelineFBP_denoising, + method=[ + "find_center_vo", + "find_center_vo", + "find_center_vo", + "total_variation_PD", + ], + key=["cor_initialisation_value", "smin", "smax", "iterations"], + value=[80.0, -20, 20, 200], + ) + + cmd.pop(4) #: don't save all + cmd.insert(6, standard_data) + cmd.insert(7, gpu_pipelineFBP_denoising) + cmd.insert(8, output_folder) + subprocess.check_output(cmd) + + # recurse through output_dir and check that all files are there + files = get_files("output_dir/") + assert len(files) == 132 + + _check_tif(files, 128, (160, 160)) + + #: check the generated h5 files + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + for file_to_open in h5_files: + if "httomolibgpu-FBP" in file_to_open: + h5f = h5py.File(file_to_open, "r") + assert h5f["data"].shape == (160, 128, 160) + assert h5f["data"].dtype == np.float32 + h5f.close() + else: + raise FileNotFoundError("File with httomolibgpu-FBP string cannot be found") + + verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) + user_log_file = list(filter(lambda x: "user.log" in x, files)) + assert len(verbose_log_file) == 1 + assert len(user_log_file) == 1 + verbose_log_contents = _get_log_contents(verbose_log_file[0]) + + assert f"{user_log_file[0]}" in verbose_log_contents + assert "The full dataset shape is (220, 128, 160)" in verbose_log_contents + assert "Loading data: tests/test_data/tomo_standard.nxs" in verbose_log_contents + assert "Path to data: /entry1/tomo_entry/data/data" in verbose_log_contents + assert "Preview: (0:180, 0:128, 0:160)" in verbose_log_contents + assert "Data shape is (180, 128, 160) of type uint16" in verbose_log_contents + assert "The amount of the available GPU memory is" in verbose_log_contents + assert ( + "Using GPU 0 to transfer data of shape (180, 128, 160)" in verbose_log_contents + ) + + +# TODO: rewrite and move to test_pipeline_big +# @pytest.mark.small_data +# def test_tomo_standard_testing_pipeline_output_with_save_all( +# get_files: Callable, +# cmd, +# standard_data, +# standard_loader, +# testing_pipeline, +# output_folder, +# merge_yamls, +# ): +# cmd.insert(7, standard_data) +# merge_yamls(standard_loader, testing_pipeline) +# cmd.insert(8, "temp.yaml") +# cmd.insert(9, output_folder) +# subprocess.check_output(cmd) + +# files = get_files("output_dir/") +# assert len(files) == 11 + +# _check_yaml(files, "temp.yaml") +# _check_tif(files, 3, (160, 160)) + +# #: check the generated h5 files +# h5_files = list(filter(lambda x: ".h5" in x, files)) +# assert len(h5_files) == 5 + +# for file_to_open in h5_files: +# if "tomopy-recon-tomo-gridrec.h5" in file_to_open: +# with h5py.File(file_to_open, "r") as f: +# assert f["data"].shape == (160, 3, 160) +# assert f["data"].dtype == np.float32 +# assert_allclose(np.mean(f["data"]), 0.0015362317, atol=1e-6, rtol=1e-6) +# assert_allclose(np.sum(f["data"]), 117.9826, atol=1e-6, rtol=1e-6) + + +# TODO: we will be testing this in test_pipeline_big +# def test_i12_testing_pipeline_output( +# get_files: Callable, +# cmd, +# i12_data, +# i12_loader, +# testing_pipeline, +# output_folder, +# merge_yamls, +# ): +# cmd.insert(7, i12_data) +# merge_yamls(i12_loader, testing_pipeline) +# cmd.insert(8, "temp.yaml") +# cmd.insert(9, output_folder) +# subprocess.check_output(cmd) + +# files = get_files("output_dir/") +# assert len(files) == 18 + +# _check_yaml(files, "temp.yaml") + +# log_files = list(filter(lambda x: ".log" in x, files)) +# assert len(log_files) == 2 + +# tif_files = list(filter(lambda x: ".tif" in x, files)) +# assert len(tif_files) == 10 + +# h5_files = list(filter(lambda x: ".h5" in x, files)) +# assert len(h5_files) == 5 + +# gridrec_recon = list(filter(lambda x: "recon-gridrec.h5" in x, h5_files))[0] +# minus_log_tomo = list(filter(lambda x: "minus_log.h5" in x, h5_files))[0] +# remove_stripe_fw_tomo = list( +# filter(lambda x: "remove_stripe_fw.h5" in x, h5_files) +# )[0] +# normalize_tomo = list(filter(lambda x: "normalize.h5" in x, h5_files))[0] + +# with h5py.File(gridrec_recon, "r") as f: +# assert f["data"].shape == (192, 10, 192) +# assert_allclose(np.sum(f["data"]), 2157.03, atol=1e-2, rtol=1e-6) +# assert_allclose(np.mean(f["data"]), 0.0058513316, atol=1e-6, rtol=1e-6) +# with h5py.File(minus_log_tomo, "r") as f: +# assert_allclose(np.sum(f["data"]), 1756628.4, atol=1e-6, rtol=1e-6) +# assert_allclose(np.mean(f["data"]), 1.2636887, atol=1e-6, rtol=1e-6) +# with h5py.File(remove_stripe_fw_tomo, "r") as f: +# assert_allclose(np.sum(f["data"]), 1766357.8, atol=1e-6, rtol=1e-6) +# assert_allclose(np.mean(f["data"]), 1.2706878, atol=1e-6, rtol=1e-6) +# with h5py.File(normalize_tomo, "r") as f: +# assert f["data"].shape == (724, 10, 192) +# assert_allclose(np.sum(f["data"]), 393510.72, atol=1e-6, rtol=1e-6) +# assert_allclose(np.mean(f["data"]), 0.28308493, atol=1e-6, rtol=1e-6) + +# concise_log_file = list(filter(lambda x: "user.log" in x, files)) +# concise_log_contents = _get_log_contents(concise_log_file[0]) +# verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) +# verbose_log_contents = _get_log_contents(verbose_log_file[0]) + +# assert "The center of rotation is 95.5" in concise_log_contents +# assert "The full dataset shape is (724, 10, 192)" in verbose_log_contents +# assert ( +# "Loading data: tests/test_data/i12/separate_flats_darks/i12_dynamic_start_stop180.nxs" +# in verbose_log_contents +# ) +# assert "Path to data: /1-TempPlugin-tomo/data" in verbose_log_contents +# assert "Preview: (0:724, 0:10, 0:192)" in verbose_log_contents + + +# TODO: Will be added to big-data tests when the sample data with separate darks and flats will be available. +# @pytest.mark.small_data +# def test_i12_testing_ignore_darks_flats_pipeline_output( +# get_files: Callable, +# cmd, +# i12_data, +# i12_loader_ignore_darks_flats, +# testing_pipeline, +# output_folder, +# merge_yamls, +# ): +# cmd.insert(7, i12_data) +# merge_yamls(i12_loader_ignore_darks_flats, testing_pipeline) +# cmd.insert(8, "temp.yaml") +# cmd.insert(9, output_folder) +# subprocess.check_output(cmd) + +# files = get_files("output_dir/") +# assert len(files) == 16 + +# _check_yaml(files, "temp.yaml") + +# log_files = list(filter(lambda x: ".log" in x, files)) +# assert len(log_files) == 1 + +# tif_files = list(filter(lambda x: ".tif" in x, files)) +# assert len(tif_files) == 10 + +# h5_files = list(filter(lambda x: ".h5" in x, files)) +# assert len(h5_files) == 4 + +# log_contents = _get_log_contents(log_files[0]) +# assert "The full dataset shape is (724, 10, 192)" in log_contents +# assert ( +# "Loading data: tests/test_data/i12/separate_flats_darks/i12_dynamic_start_stop180.nxs" +# in log_contents +# ) +# assert "Path to data: /1-TempPlugin-tomo/data" in log_contents +# assert "Preview: (0:724, 0:10, 0:192)" in log_contents +# assert ( +# "Running save_task_1 (pattern=projection): save_intermediate_data..." +# in log_contents +# ) +# assert ( +# "Running save_task_2 (pattern=projection): save_intermediate_data..." +# in log_contents +# ) +# assert ( +# "Running save_task_4 (pattern=sinogram): save_intermediate_data..." +# in log_contents +# ) +# assert "The center of rotation for sinogram is 95.5" in log_contents +# assert ( +# "Running save_task_5 (pattern=sinogram): save_intermediate_data..." +# in log_contents +# ) + + +@pytest.mark.small_data +def test_run_gpu_pipeline_sweep_cor( + get_files: Callable, cmd, standard_data, yaml_gpu_pipeline_sweep_cor, output_folder +): + cmd.pop(4) #: don't save all + cmd.insert(6, standard_data) + cmd.insert(7, yaml_gpu_pipeline_sweep_cor) + cmd.insert(8, output_folder) + subprocess.check_output(cmd) + + # recurse through output_dir and check that all files are there + files = get_files("output_dir/") + assert len(files) == 9 + + #: check the generated h5 files + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 0 + + log_files = list(filter(lambda x: ".log" in x, files)) + assert len(log_files) == 2 + verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) + verbose_log_contents = _get_log_contents(verbose_log_file[0]) + + assert "Data shape is (180, 7, 160) of type uint16" in verbose_log_contents + assert "Total number of values across all processes: 6" in verbose_log_contents + assert "Values executed in this process: 6" in verbose_log_contents + + +@pytest.mark.small_data +def test_run_gpu_pipeline_sweep_paganin( + get_files: Callable, + cmd, + standard_data, + yaml_gpu_pipeline_sweep_paganin, + output_folder, +): + cmd.pop(4) #: don't save all + cmd.insert(6, standard_data) + cmd.insert(7, yaml_gpu_pipeline_sweep_paganin) + cmd.insert(8, output_folder) + subprocess.check_output(cmd) + + # recurse through output_dir and check that all files are there + files = get_files("output_dir/") + assert len(files) == 104 + + #: check the generated h5 files + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + log_files = list(filter(lambda x: ".log" in x, files)) + assert len(log_files) == 2 + verbose_log_file = list(filter(lambda x: "debug.log" in x, files)) + verbose_log_contents = _get_log_contents(verbose_log_file[0]) + + assert "Data shape is (180, 7, 160) of type uint16" in verbose_log_contents + assert "Total number of values across all processes: 50" in verbose_log_contents + assert "Values executed in this process: 50" in verbose_log_contents + assert "Parameter name: alpha" in verbose_log_contents diff --git a/tests/test_ui_layer.py b/tests/test_ui_layer.py index d6b46e638..fe2d30fdb 100644 --- a/tests/test_ui_layer.py +++ b/tests/test_ui_layer.py @@ -12,88 +12,35 @@ # TODO: add files with invalid syntax -def test_can_read_cpu1(yaml_cpu_pipeline1): - pipline_stage_config = ui_layer.yaml_loader(yaml_cpu_pipeline1) - - assert len(pipline_stage_config) == 7 - assert pipline_stage_config[0]["method"] == "standard_tomo" - assert pipline_stage_config[0]["module_path"] == "httomo.data.hdf.loaders" - assert pipline_stage_config[1]["method"] == "normalize" - assert pipline_stage_config[1]["module_path"] == "tomopy.prep.normalize" - assert pipline_stage_config[2]["method"] == "minus_log" - assert pipline_stage_config[2]["module_path"] == "tomopy.prep.normalize" - assert pipline_stage_config[3]["method"] == "find_center_vo" - assert pipline_stage_config[3]["module_path"] == "tomopy.recon.rotation" - assert pipline_stage_config[3]["side_outputs"] == {"cor": "centre_of_rotation"} - assert pipline_stage_config[4]["method"] == "recon" - assert pipline_stage_config[5]["method"] == "rescale_to_int" - assert pipline_stage_config[5]["module_path"] == "httomolibgpu.misc.rescale" - assert pipline_stage_config[4]["module_path"] == "tomopy.recon.algorithm" - assert pipline_stage_config[6]["method"] == "save_to_images" - assert pipline_stage_config[6]["module_path"] == "httomolib.misc.images" - - -def test_can_read_cpu2(yaml_cpu_pipeline2): - pipline_stage_config = ui_layer.yaml_loader(yaml_cpu_pipeline2) - - assert len(pipline_stage_config) == 10 - assert pipline_stage_config[0]["method"] == "standard_tomo" - assert pipline_stage_config[0]["module_path"] == "httomo.data.hdf.loaders" - assert pipline_stage_config[0]["parameters"]["preview"] == { - "detector_y": {"start": 30, "stop": 60} - } - assert pipline_stage_config[1]["method"] == "find_center_vo" - assert pipline_stage_config[1]["module_path"] == "tomopy.recon.rotation" - assert pipline_stage_config[1]["side_outputs"] == {"cor": "centre_of_rotation"} - assert pipline_stage_config[2]["method"] == "remove_outlier" - assert pipline_stage_config[2]["module_path"] == "tomopy.misc.corr" - assert pipline_stage_config[3]["method"] == "normalize" - assert pipline_stage_config[3]["module_path"] == "tomopy.prep.normalize" - assert pipline_stage_config[4]["method"] == "minus_log" - assert pipline_stage_config[4]["module_path"] == "tomopy.prep.normalize" - assert pipline_stage_config[5]["method"] == "remove_stripe_fw" - assert pipline_stage_config[5]["module_path"] == "tomopy.prep.stripe" - assert pipline_stage_config[6]["method"] == "recon" - assert pipline_stage_config[6]["module_path"] == "tomopy.recon.algorithm" - assert pipline_stage_config[6]["save_result"] == True - assert pipline_stage_config[7]["method"] == "median_filter" - assert pipline_stage_config[7]["module_path"] == "tomopy.misc.corr" - assert pipline_stage_config[8]["method"] == "rescale_to_int" - assert pipline_stage_config[8]["module_path"] == "httomolibgpu.misc.rescale" - assert pipline_stage_config[9]["method"] == "save_to_images" - assert pipline_stage_config[9]["module_path"] == "httomolib.misc.images" - - -def test_can_read_cpu3(yaml_cpu_pipeline3): - pipline_stage_config = ui_layer.yaml_loader(yaml_cpu_pipeline3) +def test_can_read_cpu_pipeline(cpu_pipeline_gridrec): + pipline_stage_config = ui_layer.yaml_loader(cpu_pipeline_gridrec) assert len(pipline_stage_config) == 9 assert pipline_stage_config[0]["method"] == "standard_tomo" assert pipline_stage_config[0]["module_path"] == "httomo.data.hdf.loaders" - assert pipline_stage_config[1]["method"] == "find_center_vo" - assert pipline_stage_config[1]["module_path"] == "tomopy.recon.rotation" - assert pipline_stage_config[1]["side_outputs"] == {"cor": "centre_of_rotation"} - assert pipline_stage_config[2]["method"] == "remove_outlier" - assert pipline_stage_config[2]["module_path"] == "tomopy.misc.corr" - assert pipline_stage_config[3]["method"] == "normalize" + assert pipline_stage_config[1]["method"] == "remove_outlier" + assert pipline_stage_config[1]["module_path"] == "tomopy.misc.corr" + assert pipline_stage_config[2]["method"] == "normalize" + assert pipline_stage_config[2]["module_path"] == "tomopy.prep.normalize" + assert pipline_stage_config[3]["method"] == "minus_log" assert pipline_stage_config[3]["module_path"] == "tomopy.prep.normalize" - assert pipline_stage_config[4]["method"] == "minus_log" - assert pipline_stage_config[4]["module_path"] == "tomopy.prep.normalize" + assert pipline_stage_config[4]["method"] == "find_center_vo" + assert pipline_stage_config[4]["module_path"] == "tomopy.recon.rotation" + assert pipline_stage_config[4]["side_outputs"] == {"cor": "centre_of_rotation"} assert pipline_stage_config[5]["method"] == "recon" assert pipline_stage_config[5]["module_path"] == "tomopy.recon.algorithm" assert pipline_stage_config[6]["method"] == "calculate_stats" assert pipline_stage_config[6]["module_path"] == "httomo.methods" - assert pipline_stage_config[6]["side_outputs"] == {"glob_stats": "glob_stats"} assert pipline_stage_config[7]["method"] == "rescale_to_int" assert pipline_stage_config[7]["module_path"] == "httomolibgpu.misc.rescale" assert pipline_stage_config[8]["method"] == "save_to_images" assert pipline_stage_config[8]["module_path"] == "httomolib.misc.images" -def test_can_read_gpu1(yaml_gpu_pipeline1): - pipline_stage_config = ui_layer.yaml_loader(yaml_gpu_pipeline1) +def test_can_read_gpu_pipeline(gpu_pipelineFBP): + pipline_stage_config = ui_layer.yaml_loader(gpu_pipelineFBP) - assert len(pipline_stage_config) == 8 + assert len(pipline_stage_config) == 9 assert pipline_stage_config[0]["method"] == "standard_tomo" assert pipline_stage_config[0]["module_path"] == "httomo.data.hdf.loaders" assert pipline_stage_config[1]["method"] == "find_center_vo" @@ -103,14 +50,16 @@ def test_can_read_gpu1(yaml_gpu_pipeline1): assert pipline_stage_config[2]["module_path"] == "httomolibgpu.misc.corr" assert pipline_stage_config[3]["method"] == "normalize" assert pipline_stage_config[3]["module_path"] == "httomolibgpu.prep.normalize" - assert pipline_stage_config[4]["method"] == "remove_stripe_based_sorting" + assert pipline_stage_config[4]["method"] == "remove_all_stripe" assert pipline_stage_config[4]["module_path"] == "httomolibgpu.prep.stripe" assert pipline_stage_config[5]["method"] == "FBP" assert pipline_stage_config[5]["module_path"] == "httomolibgpu.recon.algorithm" - assert pipline_stage_config[6]["method"] == "rescale_to_int" - assert pipline_stage_config[6]["module_path"] == "httomolibgpu.misc.rescale" - assert pipline_stage_config[7]["method"] == "save_to_images" - assert pipline_stage_config[7]["module_path"] == "httomolib.misc.images" + assert pipline_stage_config[6]["method"] == "calculate_stats" + assert pipline_stage_config[6]["module_path"] == "httomo.methods" + assert pipline_stage_config[7]["method"] == "rescale_to_int" + assert pipline_stage_config[7]["module_path"] == "httomolibgpu.misc.rescale" + assert pipline_stage_config[8]["method"] == "save_to_images" + assert pipline_stage_config[8]["module_path"] == "httomolib.misc.images" @pytest.mark.parametrize("file", ["does_not_exist.yaml"]) @@ -120,9 +69,9 @@ def test_uilayer_fails_with_nonexistant_file(file: str): UiLayer(Path(file), Path("doesnt_matter"), comm=comm) -def test_pipeline_build_no_loader(yaml_cpu_pipeline1, standard_data): +def test_pipeline_build_no_loader(cpu_pipeline_gridrec, standard_data): comm = MPI.COMM_NULL - LayerUI = UiLayer(yaml_cpu_pipeline1, standard_data, comm=comm) + LayerUI = UiLayer(cpu_pipeline_gridrec, standard_data, comm=comm) del LayerUI.PipelineStageConfig[0] with pytest.raises(ValueError) as e: LayerUI.build_pipeline() @@ -130,9 +79,9 @@ def test_pipeline_build_no_loader(yaml_cpu_pipeline1, standard_data): assert "no loader" in str(e) -def test_pipeline_build_duplicate_id(yaml_cpu_pipeline1, standard_data): +def test_pipeline_build_duplicate_id(cpu_pipeline_gridrec, standard_data): comm = MPI.COMM_NULL - LayerUI = UiLayer(yaml_cpu_pipeline1, standard_data, comm=comm) + LayerUI = UiLayer(cpu_pipeline_gridrec, standard_data, comm=comm) LayerUI.PipelineStageConfig[1]["id"] = "testid" LayerUI.PipelineStageConfig[2]["id"] = "testid" with pytest.raises(ValueError) as e: @@ -141,90 +90,37 @@ def test_pipeline_build_duplicate_id(yaml_cpu_pipeline1, standard_data): assert "duplicate id" in str(e) -def test_pipeline_build_cpu1(standard_data, yaml_cpu_pipeline1): +def test_pipeline_build_cpu_pipeline(standard_data, cpu_pipeline_gridrec): """Testing OutputRef.""" comm = MPI.COMM_WORLD - LayerUI = UiLayer(yaml_cpu_pipeline1, standard_data, comm=comm) + LayerUI = UiLayer(cpu_pipeline_gridrec, standard_data, comm=comm) pipeline = LayerUI.build_pipeline() - assert len(pipeline) == 6 + assert len(pipeline) == 8 methods = [ + "remove_outlier", "normalize", "minus_log", "find_center_vo", "recon", + "calculate_stats", "rescale_to_int", "save_to_images", ] for i in range(5): assert pipeline[i].method_name == methods[i] - if i != 2: + if i != 3: assert pipeline[i].task_id == f"task_{i+1}" else: assert pipeline[i].task_id == "centering" - ref = pipeline[3]["center"] + ref = pipeline[4]["center"] assert isinstance(ref, OutputRef) assert ref.mapped_output_name == "centre_of_rotation" assert ref.method.method_name == "find_center_vo" -def test_pipeline_build_cpu2(standard_data, yaml_cpu_pipeline2): - """Testing OutputRef.""" - comm = MPI.COMM_WORLD - LayerUI = UiLayer(yaml_cpu_pipeline2, standard_data, comm=comm) - - pipeline = LayerUI.build_pipeline() - - assert len(pipeline) == 9 - methods = [ - "find_center_vo", - "remove_outlier", - "normalize", - "minus_log", - "remove_stripe_fw", - "recon", - "median_filter", - "rescale_to_int", - "save_to_images", - ] - for i in range(8): - assert pipeline[i].method_name == methods[i] - ref = pipeline[5]["center"] - assert isinstance(ref, OutputRef) - assert ref.mapped_output_name == "centre_of_rotation" - assert ref.method.method_name == "find_center_vo" - assert ref.method.task_id == "centering" - - -def test_pipeline_build_cpu3(standard_data, yaml_cpu_pipeline3): - """Testing OutputRef.""" - comm = MPI.COMM_WORLD - LayerUI = UiLayer(yaml_cpu_pipeline3, standard_data, comm=comm) - - pipeline = LayerUI.build_pipeline() - - assert len(pipeline) == 8 - methods = [ - "find_center_vo", - "remove_outlier", - "normalize", - "minus_log", - "recon", - "calculate_stats", - "rescale_to_int", - "save_to_images", - ] - for i in range(8): - assert pipeline[i].method_name == methods[i] - ref = pipeline[6]["glob_stats"] - assert isinstance(ref, OutputRef) - assert ref.mapped_output_name == "glob_stats" - assert ref.method.method_name == "calculate_stats" - assert ref.method.task_id == "statistics" - - @pytest.mark.parametrize( "pipeline_file, expected_sweep_vals", [ diff --git a/tests/test_yaml_checker.py b/tests/test_yaml_checker.py index da97d157b..d7d640def 100644 --- a/tests/test_yaml_checker.py +++ b/tests/test_yaml_checker.py @@ -147,18 +147,18 @@ def test_check_side_out_matches_ref_arg( assert check_side_out_matches_ref_arg(conf) == expected +# In order for this test to pass you'd need to generate the pipelines first +@pytest.mark.small_data @pytest.mark.parametrize( "yaml_file, expected", [ - ("pipeline_cpu1.yaml", True), - ("pipeline_cpu2.yaml", True), - ("pipeline_gpu1.yaml", True), + ("../../../docs/source/pipelines_full/cpu_pipeline_gridrec.yaml", True), + ("../../../docs/source/pipelines_full/gpu_pipelineFBP.yaml", True), ("testing/sweep_manual.yaml", True), ], ids=[ - "cpu1_pipeline", - "cpu2_pipeline", - "gpu1_pipeline", + "pipeline_cpu1", + "pipeline_gpu1", "sweep_manual", ], )