From a51bcc0adac669d2daf4a19f00fbaebb47964159 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Tue, 22 Mar 2022 15:45:29 +0000 Subject: [PATCH 01/13] Added the ability to interpret scanspecs --- setup.cfg | 1 + src/nexgen/nxs_write/__init__.py | 27 +++++++++++++--- tests/test_calculate_scan_range.py | 50 ++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 tests/test_calculate_scan_range.py diff --git a/setup.cfg b/setup.cfg index 8d2b2825..27611cd3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,7 @@ install_requires = numpy pint importlib_resources>=1.1 + scanspec packages = find: package_dir = =src diff --git a/src/nexgen/nxs_write/__init__.py b/src/nexgen/nxs_write/__init__.py index c53a7d1e..7b282e9b 100644 --- a/src/nexgen/nxs_write/__init__.py +++ b/src/nexgen/nxs_write/__init__.py @@ -8,7 +8,10 @@ from pathlib import Path from h5py import AttributeManager -from typing import List, Tuple, Union +from typing import List, Tuple, Union, Dict + +from scanspec.core import Path +from scanspec.specs import Spec, Line def create_attributes( @@ -97,6 +100,21 @@ def find_scan_axis( return scan_axis +def calculate_scan_from_scanspec(spec: Spec) -> Dict[str, np.ndarray]: + """Returns the numpy arrarys describing a scan based on a specscan object + (see https://dls-controls.github.io/scanspec/master/index.html). + + Args: + spec (Spec): A specification for the scan + + Returns: + Dict[str, np.ndarray]: A dictionary with the axis name and numpy + array describing the movement of that axis + """ + path = Path(spec.calculate()) + return path.consume().midpoints + + def calculate_scan_range( axis_start: float, axis_end: float, @@ -117,10 +135,9 @@ def calculate_scan_range( scan_range (np.ndarray): Numpy array of values for the rotation axis. """ if n_images: - if axis_start == axis_end: - scan_range = np.repeat(axis_start, n_images) - else: - scan_range = np.linspace(axis_start, axis_end, n_images) + tmp_axis_name = "" + scan_spec = Line(tmp_axis_name, axis_start, axis_end, n_images) + scan_range = calculate_scan_from_scanspec(scan_spec)[tmp_axis_name] else: scan_range = np.arange(axis_start, axis_end, axis_increment) return scan_range diff --git a/tests/test_calculate_scan_range.py b/tests/test_calculate_scan_range.py new file mode 100644 index 00000000..21d3003a --- /dev/null +++ b/tests/test_calculate_scan_range.py @@ -0,0 +1,50 @@ +from nexgen.nxs_write import calculate_scan_range, calculate_scan_from_scanspec +import numpy as np +from scanspec.specs import Line + + +def test_given_start_stop_and_increment_when_calculate_scan_range_called_then_expected_range_returned(): + start = 0 + stop = 10 + increment = 0.25 + scan_range = calculate_scan_range(start, stop, increment) + assert type(scan_range) == np.ndarray + assert len(scan_range) == 40 + assert np.amax(scan_range) == 9.75 + assert np.amin(scan_range) == 0 + assert scan_range[3] == 0.75 + + +def test_given_start_stop_and_n_images_when_calculate_scan_range_called_then_expected_range_returned(): + start = 0 + stop = 10 + n_images = 41 + scan_range = calculate_scan_range(start, stop, n_images=n_images) + assert type(scan_range) == np.ndarray + assert len(scan_range) == 41 + assert np.amax(scan_range) == 10.0 + assert np.amin(scan_range) == 0 + assert scan_range[3] == 0.75 + + +def test_given_equal_start_stop_and_n_images_when_calculate_scan_range_called_then_expected_range_returned(): + start = 2 + stop = 2 + n_images = 41 + scan_range = calculate_scan_range(start, stop, n_images=n_images) + assert type(scan_range) == np.ndarray + assert len(scan_range) == 41 + assert np.amax(scan_range) == 2.0 + assert np.amin(scan_range) == 2.0 + assert scan_range[3] == 2.0 + + +def test_calculate_from_scanspec(): + spec = Line("x", 0, 10, 41) + midpoints = calculate_scan_from_scanspec(spec) + scan_range = midpoints["x"] + assert type(scan_range) == np.ndarray + assert len(scan_range) == 41 + assert np.amax(scan_range) == 10.0 + assert np.amin(scan_range) == 0 + assert scan_range[3] == 0.75 From 7477cbf9af636c8d420f1c6e748fc61206fccf1d Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Tue, 22 Mar 2022 16:44:40 +0000 Subject: [PATCH 02/13] Added tests for various of the nexus writers --- tests/test_NxclassWriters.py | 192 +++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 tests/test_NxclassWriters.py diff --git a/tests/test_NxclassWriters.py b/tests/test_NxclassWriters.py new file mode 100644 index 00000000..f9c3f13f --- /dev/null +++ b/tests/test_NxclassWriters.py @@ -0,0 +1,192 @@ +from pathlib import Path +from nexgen.nxs_write.NXclassWriters import write_NXdata, find_scan_axis, write_NXsample +from unittest.mock import MagicMock, patch +import pytest +import tempfile +import h5py +from h5py import AttributeManager +import numpy as np +from numpy.testing import assert_array_equal + +test_goniometer_axes = { + "axes": ["omega", "sam_z", "sam_y"], + "depends": [".", "omega", "sam_z"], + "vectors": [ + -1, + 0, + 0, + 0, + -1, + 0, + -1, + 0, + 0, + ], + "types": [ + "rotation", + "translation", + "translation", + ], + "units": ["deg", "mm", "mm"], + "offsets": [0, 0, 0, 0, 0, 0, 0, 0, 0], + "starts": [0, 0, 0], + "ends": [0, 0, 0], + "increments": [1, 0, 0], +} + + +@pytest.fixture +def dummy_nexus_file(): + test_hdf_file = tempfile.TemporaryFile() + test_nexus_file = h5py.File(test_hdf_file, "w") + yield test_nexus_file + + +def test_given_no_data_files_when_write_NXdata_then_assert_error(): + mock_hdf5_file = MagicMock() + with pytest.raises(AssertionError): + write_NXdata(mock_hdf5_file, [], {}, "", "", []) + + +def test_given_no_data_type_specified_when_write_NXdata_then_exception_raised( + dummy_nexus_file, +): + with pytest.raises(SystemExit): + write_NXdata( + dummy_nexus_file, [Path("tmp")], test_goniometer_axes, "", "", [], "sam_z" + ) + + +def test_given_one_data_file_when_write_NXdata_then_data_entry_in_file( + dummy_nexus_file, +): + write_NXdata( + dummy_nexus_file, [Path("tmp")], test_goniometer_axes, "images", "", [], "sam_z" + ) + assert dummy_nexus_file["/entry/data"].attrs["NX_class"] == b"NXdata" + assert "data" in dummy_nexus_file["/entry/data"] + + +@patch("nexgen.nxs_write.NXclassWriters.find_scan_axis", return_value="sam_z") +def test_given_no_scan_axis_when_write_NXdata_then_find_scan_axis_called( + mock_find_scan_axis, dummy_nexus_file +): + write_NXdata( + dummy_nexus_file, [Path("tmp")], test_goniometer_axes, "images", "", [] + ) + mock_find_scan_axis.assert_called_once() + + +def test_given_scan_axis_when_write_NXdata_then_axis_in_data_entry_with_correct_data_and_attributes( + dummy_nexus_file, +): + test_axis = "sam_z" + test_scan_range = [0, 1, 2] + axis_entry = f"/entry/data/{test_axis}" + + write_NXdata( + dummy_nexus_file, + [Path("tmp")], + test_goniometer_axes, + "images", + "", + test_scan_range, + test_axis, + ) + + assert test_axis in dummy_nexus_file["/entry/data"] + assert_array_equal(test_scan_range, dummy_nexus_file[axis_entry][:]) + assert ( + dummy_nexus_file[axis_entry].attrs["depends_on"] + == b"/entry/sample/transformations/omega" + ) + assert dummy_nexus_file[axis_entry].attrs["transformation_type"] == b"translation" + assert dummy_nexus_file[axis_entry].attrs["units"] == b"mm" + assert_array_equal(dummy_nexus_file[axis_entry].attrs["vector"][:], [0.0, -1.0, 0]) + + +def test_given_scan_axis_when_write_NXsample_then_scan_axis_data_copied_from_data_group_as_well_as_increment_set_and_end( + dummy_nexus_file, +): + test_axis = "omega" + test_scan_range = [0, 1, 2] + axis_entry = f"/entry/sample/sample_{test_axis}/{test_axis}" + + # Doing this to write the scan axis data into the data group + write_NXdata( + dummy_nexus_file, + [Path("tmp")], + test_goniometer_axes, + "images", + "", + test_scan_range, + test_axis, + ) + + write_NXsample( + dummy_nexus_file, + test_goniometer_axes, + "", + "images", + test_axis, + test_scan_range, + ) + + assert f"sample_{test_axis}" in dummy_nexus_file["/entry/sample"] + assert_array_equal(test_scan_range, dummy_nexus_file[axis_entry][:]) + assert dummy_nexus_file[axis_entry].attrs["depends_on"] == b"." + assert dummy_nexus_file[axis_entry].attrs["transformation_type"] == b"rotation" + assert dummy_nexus_file[axis_entry].attrs["units"] == b"deg" + assert_array_equal(dummy_nexus_file[axis_entry].attrs["vector"][:], [-1, 0, 0]) + assert_array_equal(dummy_nexus_file[axis_entry + "_increment_set"][:], [1] * 3) + assert dummy_nexus_file[axis_entry + "_end"][1] == 2 + + +def test_given_no_axes_when_find_scan_axis_called_then_assert_error(): + with pytest.raises(AssertionError): + find_scan_axis([], [], [], []) + + +def test_given_one_rotation_axis_when_find_scan_axis_called_then_axis_returned(): + test_names = ["sam_x", "omega"] + test_starts = [0, 0] + test_ends = [0, 10] + test_types = ["translation", "rotation"] + scan_axis = find_scan_axis(test_names, test_starts, test_ends, test_types) + assert scan_axis == "omega" + + +def test_given_no_moving_axes_when_find_scan_axis_called_then_default_axis_returned(): + test_names = ["sam_x", "omega"] + test_starts = [0, 0] + test_ends = [0, 0] + test_types = ["rotation", "rotation"] + default_axis = "default_axis" + scan_axis = find_scan_axis( + test_names, test_starts, test_ends, test_types, default_axis + ) + assert scan_axis == default_axis + + +def test_given_one_moving_axes_when_find_scan_axis_called_then_this_axis_returned(): + test_names = ["sam_x", "omega"] + test_starts = [0, 0] + test_ends = [0, 10] + test_types = ["rotation", "rotation"] + default_axis = "default_axis" + scan_axis = find_scan_axis( + test_names, test_starts, test_ends, test_types, default_axis + ) + assert scan_axis == "omega" + + +def test_given_two_moving_axes_when_find_scan_axis_called_then_exception(): + test_names = ["sam_x", "omega"] + test_starts = [0, 0] + test_ends = [10, 10] + test_types = ["rotation", "rotation"] + default_axis = "default_axis" + with pytest.raises(SystemExit): + scan_axis = find_scan_axis( + test_names, test_starts, test_ends, test_types, default_axis + ) From 629ad0a2e68960752cff5c16cd02df4b52163b0c Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Tue, 22 Mar 2022 17:50:54 +0000 Subject: [PATCH 03/13] Changed scan_range to be a dictionary with the axis name as the range --- src/nexgen/beamlines/I24_Eiger_nxs.py | 13 +-- src/nexgen/beamlines/SSX_Tristan_nxs.py | 4 +- src/nexgen/nxs_write/NXclassWriters.py | 106 +++++++++++++----------- src/nexgen/nxs_write/NexusWriter.py | 15 ++-- tests/test_NxclassWriters.py | 65 ++++++++++----- 5 files changed, 117 insertions(+), 86 deletions(-) diff --git a/src/nexgen/beamlines/I24_Eiger_nxs.py b/src/nexgen/beamlines/I24_Eiger_nxs.py index 3165cdf0..f76d3a16 100644 --- a/src/nexgen/beamlines/I24_Eiger_nxs.py +++ b/src/nexgen/beamlines/I24_Eiger_nxs.py @@ -124,11 +124,13 @@ def extruder( goniometer["types"], ) scan_idx = goniometer["axes"].index(scan_axis) - scan_range = calculate_scan_range( - goniometer["starts"][scan_idx], - goniometer["ends"][scan_idx], - n_images=SSX.num_imgs, - ) + scan_range = { + scan_axis: calculate_scan_range( + goniometer["starts"][scan_idx], + goniometer["ends"][scan_idx], + n_images=SSX.num_imgs, + ) + } logger.info("Goniometer information") for j in range(len(goniometer["axes"])): @@ -147,7 +149,6 @@ def extruder( nxsfile, filename, "mcstas", - scan_axis, # This should be omega scan_range, (detector["mode"], SSX.num_imgs), goniometer, diff --git a/src/nexgen/beamlines/SSX_Tristan_nxs.py b/src/nexgen/beamlines/SSX_Tristan_nxs.py index 5c5042a7..01067f2b 100644 --- a/src/nexgen/beamlines/SSX_Tristan_nxs.py +++ b/src/nexgen/beamlines/SSX_Tristan_nxs.py @@ -122,8 +122,7 @@ def write_nxs(**ssx_params): ] # Get scan range array and rotation axis - scan_axis = "phi" - scan_range = (0.0, 0.0) + scan_range = {"phi": (0.0, 0.0)} # scan_axis = find_scan_axis( # goniometer["axes"], # goniometer["starts"], @@ -187,7 +186,6 @@ def write_nxs(**ssx_params): nxsfile, [metafile], "mcstas", - scan_axis, scan_range, ( detector["mode"], diff --git a/src/nexgen/nxs_write/NXclassWriters.py b/src/nexgen/nxs_write/NXclassWriters.py index 5f368a20..d5f489bd 100644 --- a/src/nexgen/nxs_write/NXclassWriters.py +++ b/src/nexgen/nxs_write/NXclassWriters.py @@ -61,8 +61,7 @@ def write_NXdata( goniometer: Dict, data_type: str, coord_frame: str, - scan_range: Union[Tuple, np.ndarray], - scan_axis: str = None, + scan_range: Dict[str, Union[Tuple, np.ndarray]], write_vds: str = None, ): """ @@ -74,37 +73,38 @@ def write_NXdata( goniometer (Dict): Dictionary containing all the axes information data_type (str): Images or events coord_frame (str): Coordinate system the axes are currently in - scan_range (Tuple|array): If writing events, this is just a (start, end) tuple - scan_axis (str): Rotation axis + scan_range (Dict): A dictionary of the scan axes and their movements write_vds (str): If not None, writes a Virtual Dataset. """ NXclass_logger.info("Start writing NXdata.") # Check that a valid datafile_list has been passed. assert len(datafiles) > 0, "Please pass at least a list of one HDF5 data file." - # If scan_axis hasn't been passed, identify it. - if not scan_axis: - scan_axis = find_scan_axis( - goniometer["axes"], - goniometer["starts"], - goniometer["ends"], - goniometer["types"], - ) - # Create NXdata group, unless it already exists, in which case just open it. nxdata = nxsfile.require_group("/entry/data") create_attributes( nxdata, - ("NX_class", "axes", "signal", scan_axis + "_indices"), + ("NX_class", "signal"), ( "NXdata", - scan_axis, "data", - [ - 0, - ], ), ) + if len(scan_range.keys()) == 1: + scan_axis = list(scan_range.keys())[0] + create_attributes( + nxdata, + ("axes", scan_axis + "_indices"), + ( + scan_axis, + [ + 0, + ], + ), + ) + else: + # I'm not sure what to do here + pass # If mode is images, link to blank image data. Else go to events. if data_type == "images": @@ -136,25 +136,26 @@ def write_NXdata( else: sys.exit("Please pass a correct data_type (images or events)") - # Write rotation axis dataset - ax = nxdata.create_dataset(scan_axis, data=scan_range) - idx = goniometer["axes"].index(scan_axis) - _dep = set_dependency( - goniometer["depends"][idx], path="/entry/sample/transformations/" - ) + for scan_axis, scan_range in scan_range.items(): + # Write rotation axis dataset + ax = nxdata.create_dataset(scan_axis, data=scan_range) + idx = goniometer["axes"].index(scan_axis) + _dep = set_dependency( + goniometer["depends"][idx], path="/entry/sample/transformations/" + ) - vectors = split_arrays(coord_frame, goniometer["axes"], goniometer["vectors"]) - # Write attributes for axis - create_attributes( - ax, - ("depends_on", "transformation_type", "units", "vector"), - ( - _dep, - goniometer["types"][idx], - goniometer["units"][idx], - vectors[scan_axis], - ), - ) + vectors = split_arrays(coord_frame, goniometer["axes"], goniometer["vectors"]) + # Write attributes for axis + create_attributes( + ax, + ("depends_on", "transformation_type", "units", "vector"), + ( + _dep, + goniometer["types"][idx], + goniometer["units"][idx], + vectors[scan_axis], + ), + ) # NXsample @@ -163,8 +164,7 @@ def write_NXsample( goniometer: Dict, coord_frame: str, data_type: str, - scan_axis: str, - scan_range: Union[Tuple, np.ndarray] = None, + scan_range: Dict[str, Union[Tuple, np.ndarray]], ): """ Write NXsample group at entry/sample @@ -174,8 +174,7 @@ def write_NXsample( goniometer (Dict): Dictionary containing all the axes information coord_frame (str): Coordinate system the axes are currently expressed in data_type (str): Images or events - scan_axis (str): Rotation axis - scan_range (Tuple|array): List/tuple/array of scan axis values + scan_range (Dict): Dictionary of scan axes and their ranges """ NXclass_logger.info("Start writing NXsample and NXtransformations.") # Create NXsample group, unless it already exists, in which case just open it. @@ -194,10 +193,15 @@ def write_NXsample( ("NXtransformations",), ) - # Save sample depends_on - nxsample.create_dataset( - "depends_on", data=set_dependency(scan_axis, path=nxtransformations.name) - ) + if len(scan_range.keys()) == 1: + # Save sample depends_on + nxsample.create_dataset( + "depends_on", + data=set_dependency(list(scan_range.keys())[0], path=nxtransformations.name), + ) + else: + # I'm not sure what this should be? + pass # Create sample_{axisname} groups vectors = split_arrays(coord_frame, goniometer["axes"], goniometer["vectors"]) @@ -208,9 +212,9 @@ def write_NXsample( grp_name = "sample_" + ax nxsample_ax = nxsample.create_group(grp_name) create_attributes(nxsample_ax, ("NX_class",), ("NXpositioner",)) - if ax == scan_axis: + if ax in scan_range.keys(): # If we're dealing with the scan axis - idx = goniometer["axes"].index(scan_axis) + idx = goniometer["axes"].index(ax) try: for k in nxsfile["/entry/data"].keys(): if isinstance( @@ -222,7 +226,7 @@ def write_NXsample( nxsample_ax[ax] = nxsfile[nxsfile["/entry/data"][k].name] nxtransformations[ax] = nxsfile[nxsfile["/entry/data"][k].name] except KeyError: - nxax = nxsample_ax.create_dataset(ax, data=scan_range) + nxax = nxsample_ax.create_dataset(ax, data=scan_range[ax]) _dep = set_dependency( goniometer["depends"][idx], path="/entry/sample/transformations/" ) @@ -233,7 +237,7 @@ def write_NXsample( _dep, goniometer["types"][idx], goniometer["units"][idx], - vectors[scan_axis], + vectors[ax], ), ) nxtransformations[ax] = nxsfile[nxax.name] @@ -241,10 +245,12 @@ def write_NXsample( # Write {axisname}_increment_set and {axis_name}_end datasets if data_type == "images": increment_set = np.repeat( - goniometer["increments"][idx], len(scan_range) + goniometer["increments"][idx], len(scan_range[ax]) ) nxsample_ax.create_dataset(ax + "_increment_set", data=increment_set) - nxsample_ax.create_dataset(ax + "_end", data=scan_range + increment_set) + nxsample_ax.create_dataset( + ax + "_end", data=scan_range[ax] + increment_set + ) else: # For all other axes idx = goniometer["axes"].index(ax) diff --git a/src/nexgen/nxs_write/NexusWriter.py b/src/nexgen/nxs_write/NexusWriter.py index 68680032..3b6d4897 100644 --- a/src/nexgen/nxs_write/NexusWriter.py +++ b/src/nexgen/nxs_write/NexusWriter.py @@ -112,12 +112,13 @@ def write_nexus( nxentry = write_NXentry(nxsfile) + scan_range = {osc_axis: scan_range} + # Call the writers call_writers( nxsfile, datafiles, coordinate_frame, - osc_axis, scan_range, data_type, goniometer.__dict__, @@ -168,7 +169,7 @@ def write_nexus_demo( This function writes a new nexus file from the information contained in the phil scopes passed as input. It also writes a specified number of blank data HDF5 files. - The nuber of these files can be passed as input parameter, if it isn't it defaults to 1. + The number of these files can be passed as input parameter, if it isn't it defaults to 1. Args: nxsfile: NeXus file to be written. @@ -244,12 +245,13 @@ def write_nexus_demo( write_NXentry(nxsfile) + scan_range = {osc_axis: scan_range} + # Call the writers call_writers( nxsfile, datafiles, coordinate_frame, - osc_axis, scan_range, data_type, goniometer.__dict__, @@ -281,8 +283,7 @@ def call_writers( nxsfile: h5py.File, datafiles: List[Path], coordinate_frame: str, - scan_axis: str, - scan_range: Union[Tuple, np.ndarray], + scan_range: Dict[str, Union[Tuple, np.ndarray]], data_type: Tuple[str, int], goniometer: Dict, detector: Dict, @@ -294,7 +295,7 @@ def call_writers( metafile: Path = None, link_list: List = None, ): - """ Call the writers for the NeXus base classes.""" + """Call the writers for the NeXus base classes.""" logger = logging.getLogger("NeXusGenerator.writer.call") logger.info("Calling the writers ...") @@ -306,7 +307,6 @@ def call_writers( data_type[0], coordinate_frame, scan_range, - scan_axis, vds, ) @@ -347,6 +347,5 @@ def call_writers( goniometer, coordinate_frame, data_type[0], - scan_axis, scan_range, ) diff --git a/tests/test_NxclassWriters.py b/tests/test_NxclassWriters.py index f9c3f13f..09441649 100644 --- a/tests/test_NxclassWriters.py +++ b/tests/test_NxclassWriters.py @@ -45,7 +45,7 @@ def dummy_nexus_file(): def test_given_no_data_files_when_write_NXdata_then_assert_error(): mock_hdf5_file = MagicMock() with pytest.raises(AssertionError): - write_NXdata(mock_hdf5_file, [], {}, "", "", []) + write_NXdata(mock_hdf5_file, [], {}, "", "", {"sam_z": []}) def test_given_no_data_type_specified_when_write_NXdata_then_exception_raised( @@ -53,7 +53,7 @@ def test_given_no_data_type_specified_when_write_NXdata_then_exception_raised( ): with pytest.raises(SystemExit): write_NXdata( - dummy_nexus_file, [Path("tmp")], test_goniometer_axes, "", "", [], "sam_z" + dummy_nexus_file, [Path("tmp")], test_goniometer_axes, "", "", {"sam_z": []} ) @@ -61,22 +61,17 @@ def test_given_one_data_file_when_write_NXdata_then_data_entry_in_file( dummy_nexus_file, ): write_NXdata( - dummy_nexus_file, [Path("tmp")], test_goniometer_axes, "images", "", [], "sam_z" + dummy_nexus_file, + [Path("tmp")], + test_goniometer_axes, + "images", + "", + {"sam_z": []}, ) assert dummy_nexus_file["/entry/data"].attrs["NX_class"] == b"NXdata" assert "data" in dummy_nexus_file["/entry/data"] -@patch("nexgen.nxs_write.NXclassWriters.find_scan_axis", return_value="sam_z") -def test_given_no_scan_axis_when_write_NXdata_then_find_scan_axis_called( - mock_find_scan_axis, dummy_nexus_file -): - write_NXdata( - dummy_nexus_file, [Path("tmp")], test_goniometer_axes, "images", "", [] - ) - mock_find_scan_axis.assert_called_once() - - def test_given_scan_axis_when_write_NXdata_then_axis_in_data_entry_with_correct_data_and_attributes( dummy_nexus_file, ): @@ -90,11 +85,11 @@ def test_given_scan_axis_when_write_NXdata_then_axis_in_data_entry_with_correct_ test_goniometer_axes, "images", "", - test_scan_range, - test_axis, + {test_axis: test_scan_range}, ) assert test_axis in dummy_nexus_file["/entry/data"] + assert dummy_nexus_file["/entry/data"].attrs["axes"] == b"sam_z" assert_array_equal(test_scan_range, dummy_nexus_file[axis_entry][:]) assert ( dummy_nexus_file[axis_entry].attrs["depends_on"] @@ -105,6 +100,40 @@ def test_given_scan_axis_when_write_NXdata_then_axis_in_data_entry_with_correct_ assert_array_equal(dummy_nexus_file[axis_entry].attrs["vector"][:], [0.0, -1.0, 0]) +def test_given_multiple_scan_axes_when_write_NXdata_then_axis_in_data_entry_with_correct_data_and_attributes( + dummy_nexus_file, +): + test_scan = {"sam_z": [0, 1, 2], "omega": [3, 4, 5]} + + write_NXdata( + dummy_nexus_file, + [Path("tmp")], + test_goniometer_axes, + "images", + "", + test_scan, + ) + + axis_entry = f"/entry/data/sam_z" + assert "sam_z" in dummy_nexus_file["/entry/data"] + assert_array_equal(test_scan["sam_z"], dummy_nexus_file[axis_entry][:]) + assert ( + dummy_nexus_file[axis_entry].attrs["depends_on"] + == b"/entry/sample/transformations/omega" + ) + assert dummy_nexus_file[axis_entry].attrs["transformation_type"] == b"translation" + assert dummy_nexus_file[axis_entry].attrs["units"] == b"mm" + assert_array_equal(dummy_nexus_file[axis_entry].attrs["vector"][:], [0.0, -1.0, 0]) + + axis_entry = f"/entry/data/omega" + assert "omega" in dummy_nexus_file["/entry/data"] + assert_array_equal(test_scan["omega"], dummy_nexus_file[axis_entry][:]) + assert dummy_nexus_file[axis_entry].attrs["depends_on"] == b"." + assert dummy_nexus_file[axis_entry].attrs["transformation_type"] == b"rotation" + assert dummy_nexus_file[axis_entry].attrs["units"] == b"deg" + assert_array_equal(dummy_nexus_file[axis_entry].attrs["vector"][:], [-1.0, 0.0, 0]) + + def test_given_scan_axis_when_write_NXsample_then_scan_axis_data_copied_from_data_group_as_well_as_increment_set_and_end( dummy_nexus_file, ): @@ -119,8 +148,7 @@ def test_given_scan_axis_when_write_NXsample_then_scan_axis_data_copied_from_dat test_goniometer_axes, "images", "", - test_scan_range, - test_axis, + {test_axis: test_scan_range}, ) write_NXsample( @@ -128,8 +156,7 @@ def test_given_scan_axis_when_write_NXsample_then_scan_axis_data_copied_from_dat test_goniometer_axes, "", "images", - test_axis, - test_scan_range, + {test_axis: test_scan_range}, ) assert f"sample_{test_axis}" in dummy_nexus_file["/entry/sample"] From 18352ed4510efac03d69ffd911c85890a1665c5b Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 23 Mar 2022 14:46:37 +0000 Subject: [PATCH 04/13] Simplified code for writing sample scan axes --- src/nexgen/nxs_write/NXclassWriters.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/nexgen/nxs_write/NXclassWriters.py b/src/nexgen/nxs_write/NXclassWriters.py index d5f489bd..b6333bb9 100644 --- a/src/nexgen/nxs_write/NXclassWriters.py +++ b/src/nexgen/nxs_write/NXclassWriters.py @@ -197,7 +197,9 @@ def write_NXsample( # Save sample depends_on nxsample.create_dataset( "depends_on", - data=set_dependency(list(scan_range.keys())[0], path=nxtransformations.name), + data=set_dependency( + list(scan_range.keys())[0], path=nxtransformations.name + ), ) else: # I'm not sure what this should be? @@ -216,15 +218,8 @@ def write_NXsample( # If we're dealing with the scan axis idx = goniometer["axes"].index(ax) try: - for k in nxsfile["/entry/data"].keys(): - if isinstance( - nxsfile["/entry/data"].get(k, getlink=True), h5py.ExternalLink - ): - # Don't even try to open! - continue - if nxsfile["/entry/data"][k].attrs.get("depends_on"): - nxsample_ax[ax] = nxsfile[nxsfile["/entry/data"][k].name] - nxtransformations[ax] = nxsfile[nxsfile["/entry/data"][k].name] + nxsample_ax[ax] = nxsfile[nxsfile["/entry/data"][ax].name] + nxtransformations[ax] = nxsfile[nxsfile["/entry/data"][ax].name] except KeyError: nxax = nxsample_ax.create_dataset(ax, data=scan_range[ax]) _dep = set_dependency( From 6d2922bc6a8e07b03db1f5dfcb137331c211ee1a Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 23 Mar 2022 14:47:08 +0000 Subject: [PATCH 05/13] Added example for writing a 2D gridscan --- .../beamlines/I03_2d_grid_scan_example.py | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 src/nexgen/beamlines/I03_2d_grid_scan_example.py diff --git a/src/nexgen/beamlines/I03_2d_grid_scan_example.py b/src/nexgen/beamlines/I03_2d_grid_scan_example.py new file mode 100644 index 00000000..76e0b79e --- /dev/null +++ b/src/nexgen/beamlines/I03_2d_grid_scan_example.py @@ -0,0 +1,175 @@ +""" +Define beamline parameters for I03, Eiger detector and give an example of writing a gridscan. +""" +import sys + +# import json +import glob +import h5py +import logging +from scanspec.specs import Line +import numpy as np + +from typing import List +from pathlib import Path +import os +from nexgen.nxs_write import calculate_scan_from_scanspec + +from nexgen.nxs_write.NexusWriter import call_writers +from nexgen.nxs_write.NXclassWriters import write_NXentry, write_NXnote + +from nexgen.tools.VDS_tools import image_vds_writer + + +source = { + "name": "Diamond Light Source", + "short_name": "DLS", + "type": "Synchrotron X-ray Source", + "beamline_name": "I03", +} + + +# fmt: off +goniometer_axes = { + "axes": ["omega", "sam_z", "sam_y", "sam_x", "chi", "phi"], + "depends": [".", "omega", "sam_z", "sam_y", "sam_x", "chi"], + "vectors": [ + -1, 0.0, 0.0, + 0.0, 0.0, 1.0, + 0.0, 1.0, 0.0, + 1.0, 0.0, 0.0, + 0.006, -0.0264, 0.9996 + -1, -0.0025, -0.0056, + ], + "types": [ + "rotation", + "translation", + "translation", + "translation", + "rotation", + "rotation", + ], + "units": ["deg", "mm", "mm", "mm", "deg", "deg"], + "offsets": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "starts": [150.0, 0.0, None, None, 0.0, 0.0], + "ends": [0.0] * 6, + "increments": [0.0] * 6, +} +# fmt: on + +eiger16M_params = { + "mode": "images", + "description": "Eiger 16M", + "detector_type": "Pixel", + "sensor_material": "Silicon", + "sensor_thickness": "4.5E-4", + "overload": 46051, + "underload": -1, # Not sure of this + "pixel_size": ["0.075mm", "0.075mm"], + "flatfield": "flatfield", + "flatfield_applied": "_dectris/flatfield_correction_applied", + "pixel_mask": "mask", + "pixel_mask_applied": "_dectris/pixel_mask_applied", + "image_size": [2068, 2162], # (fast, slow) + "axes": ["det_z"], + "depends": ["."], + "vectors": [0.0, 0.0, 1.0], + "types": ["translation"], + "units": ["mm"], + "starts": [391.228416716], + "ends": [391.228416716], + "increments": [0.0], + "bit_depth_readout": "_dectris/bit_depth_readout", + "detector_readout_time": "_dectris/detector_readout_time", + "threshold_energy": "_dectris/threshold_energy", + "software_version": "_dectris/software_version", + "serial_number": "_dectris/detector_number", + "beam_center": [1062.4015611483892, 1105.631937699275], + "exposure_time": 0.004, +} + +dset_links = [ + [ + "pixel_mask", + "pixel_mask_applied", + "flatfield", + "flatfield_applied", + "threshold_energy", + "bit_depth_readout", + "detector_readout_time", + "serial_number", + ], + ["software_version"], +] + +# Initialize dictionaries +goniometer = goniometer_axes +detector = eiger16M_params +module = { + "fast_axis": [-1.0, 0.0, 0.0], + "slow_axis": [0.0, -1.0, 0.0], + "module_offset": "1", +} +beam = {"wavelength": 0.976253543307, "flux": 9.475216962184312e11} +attenuator = {"transmission": 0.4997258186340332} + + +def example_nexus_file(): + """ + Creates an example nexus file that should closely match /dls/i03/data/2022/cm31105-1/xraycentring/TestProteinaseK/protk_1/protk_1_1.nxs + """ + + # Get timestamps in the correct format + timestamps = ( + "2022-01-19T11:23:50Z", + "2022-01-19T11:24:01Z", + ) + + # Get scan array + scan_spec = Line("sam_y", 186.5867, 366.5867, 10) * ~Line( + "sam_x", 420.9968, 1040.9968, 32 + ) + scan_range = calculate_scan_from_scanspec(scan_spec) + + containing_foler = Path("/scratch/ffv81422/artemis/example data/protk_1") + test_nexus_file = containing_foler / "protk_1_1_test.nxs" + image_data = [containing_foler / "protk_1_1_000001.h5"] + metafile = containing_foler / "protk_1_1_meta.h5" + + if test_nexus_file.exists(): + print("File exists") + os.remove(test_nexus_file) + + with h5py.File(test_nexus_file, "x") as nxsfile: + nxentry = write_NXentry(nxsfile) + + if timestamps[0]: + nxentry.create_dataset("start_time", data=np.string_(timestamps[0])) + + call_writers( + nxsfile, + image_data, + "mcstas", + scan_range, + ("images", 320), + goniometer, + detector, + module, + source, + beam, + attenuator, + vds="dataset", + metafile=metafile, + link_list=dset_links, + ) + + # Write VDS + image_vds_writer( + nxsfile, (320, detector["image_size"][1], detector["image_size"][0]) + ) + + if timestamps[1]: + nxentry.create_dataset("end_time", data=np.string_(timestamps[1])) + + +example_nexus_file() From 9737961c2b839eeaa322a52dfdf0ff389c17e816 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 23 Mar 2022 16:29:11 +0000 Subject: [PATCH 06/13] Fixed typo in gridscan test --- src/nexgen/beamlines/I03_2d_grid_scan_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nexgen/beamlines/I03_2d_grid_scan_example.py b/src/nexgen/beamlines/I03_2d_grid_scan_example.py index 76e0b79e..0a02915c 100644 --- a/src/nexgen/beamlines/I03_2d_grid_scan_example.py +++ b/src/nexgen/beamlines/I03_2d_grid_scan_example.py @@ -38,7 +38,7 @@ 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, - 0.006, -0.0264, 0.9996 + 0.006, -0.0264, 0.9996, -1, -0.0025, -0.0056, ], "types": [ @@ -132,7 +132,7 @@ def example_nexus_file(): scan_range = calculate_scan_from_scanspec(scan_spec) containing_foler = Path("/scratch/ffv81422/artemis/example data/protk_1") - test_nexus_file = containing_foler / "protk_1_1_test.nxs" + test_nexus_file = containing_foler / "protk_1_1_nexgen.nxs" image_data = [containing_foler / "protk_1_1_000001.h5"] metafile = containing_foler / "protk_1_1_meta.h5" From 6853f481bc46cd7e8617a2398e46ac6e8594b37c Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 23 Mar 2022 16:32:09 +0000 Subject: [PATCH 07/13] Remove unused imports --- src/nexgen/beamlines/I03_2d_grid_scan_example.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/nexgen/beamlines/I03_2d_grid_scan_example.py b/src/nexgen/beamlines/I03_2d_grid_scan_example.py index 0a02915c..ee86c59f 100644 --- a/src/nexgen/beamlines/I03_2d_grid_scan_example.py +++ b/src/nexgen/beamlines/I03_2d_grid_scan_example.py @@ -1,22 +1,18 @@ """ Define beamline parameters for I03, Eiger detector and give an example of writing a gridscan. """ -import sys # import json -import glob import h5py -import logging from scanspec.specs import Line import numpy as np -from typing import List from pathlib import Path import os from nexgen.nxs_write import calculate_scan_from_scanspec from nexgen.nxs_write.NexusWriter import call_writers -from nexgen.nxs_write.NXclassWriters import write_NXentry, write_NXnote +from nexgen.nxs_write.NXclassWriters import write_NXentry from nexgen.tools.VDS_tools import image_vds_writer From 9f502338309beec77dae5da2b49464e40ecc28bb Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 23 Mar 2022 16:58:20 +0000 Subject: [PATCH 08/13] Fixed more static analysis issues --- src/nexgen/nxs_write/NXclassWriters.py | 1 - src/nexgen/nxs_write/__init__.py | 4 ++-- tests/test_NxclassWriters.py | 8 ++------ 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/nexgen/nxs_write/NXclassWriters.py b/src/nexgen/nxs_write/NXclassWriters.py index b6333bb9..ee70d951 100644 --- a/src/nexgen/nxs_write/NXclassWriters.py +++ b/src/nexgen/nxs_write/NXclassWriters.py @@ -12,7 +12,6 @@ from typing import List, Dict, Tuple, Union, Optional from . import ( - find_scan_axis, calculate_origin, create_attributes, set_dependency, diff --git a/src/nexgen/nxs_write/__init__.py b/src/nexgen/nxs_write/__init__.py index 7b282e9b..b58f5640 100644 --- a/src/nexgen/nxs_write/__init__.py +++ b/src/nexgen/nxs_write/__init__.py @@ -10,7 +10,7 @@ from h5py import AttributeManager from typing import List, Tuple, Union, Dict -from scanspec.core import Path +from scanspec.core import Path as ScanPath from scanspec.specs import Spec, Line @@ -111,7 +111,7 @@ def calculate_scan_from_scanspec(spec: Spec) -> Dict[str, np.ndarray]: Dict[str, np.ndarray]: A dictionary with the axis name and numpy array describing the movement of that axis """ - path = Path(spec.calculate()) + path = ScanPath(spec.calculate()) return path.consume().midpoints diff --git a/tests/test_NxclassWriters.py b/tests/test_NxclassWriters.py index 09441649..19ef32d8 100644 --- a/tests/test_NxclassWriters.py +++ b/tests/test_NxclassWriters.py @@ -1,11 +1,9 @@ from pathlib import Path from nexgen.nxs_write.NXclassWriters import write_NXdata, find_scan_axis, write_NXsample -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import pytest import tempfile import h5py -from h5py import AttributeManager -import numpy as np from numpy.testing import assert_array_equal test_goniometer_axes = { @@ -214,6 +212,4 @@ def test_given_two_moving_axes_when_find_scan_axis_called_then_exception(): test_types = ["rotation", "rotation"] default_axis = "default_axis" with pytest.raises(SystemExit): - scan_axis = find_scan_axis( - test_names, test_starts, test_ends, test_types, default_axis - ) + find_scan_axis(test_names, test_starts, test_ends, test_types, default_axis) From cf425ba3b1c5fb4f67c4374e458878aebdcff5dc Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 30 Mar 2022 12:23:31 +0100 Subject: [PATCH 09/13] Fix bad import in unit tests --- tests/test_NxclassWriters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_NxclassWriters.py b/tests/test_NxclassWriters.py index 19ef32d8..a41db243 100644 --- a/tests/test_NxclassWriters.py +++ b/tests/test_NxclassWriters.py @@ -1,5 +1,6 @@ from pathlib import Path -from nexgen.nxs_write.NXclassWriters import write_NXdata, find_scan_axis, write_NXsample +from nexgen.nxs_write import find_scan_axis +from nexgen.nxs_write.NXclassWriters import write_NXdata, write_NXsample from unittest.mock import MagicMock import pytest import tempfile From f36c685cb58058671666bb7096af4a6c8ce6373d Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 30 Mar 2022 13:01:15 +0100 Subject: [PATCH 10/13] Updated I19_2 script for different scan format --- src/nexgen/beamlines/I19_2_nxs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/nexgen/beamlines/I19_2_nxs.py b/src/nexgen/beamlines/I19_2_nxs.py index 7d333985..f0b0d47f 100644 --- a/src/nexgen/beamlines/I19_2_nxs.py +++ b/src/nexgen/beamlines/I19_2_nxs.py @@ -152,6 +152,8 @@ def tristan_writer( detector["flatfield"] = flatfieldfile # If these two could instead be passed, I'd be happier... + scan_range = {scan_axis: scan_range} + # Get on with the writing now... try: with h5py.File(master_file, "x") as nxsfile: @@ -164,7 +166,6 @@ def tristan_writer( nxsfile, [TR.meta_file], coordinate_frame, - scan_axis, # This should be omega scan_range, (detector["mode"], None), goniometer, @@ -222,6 +223,8 @@ def eiger_writer( n_images=n_frames, ) + scan_range = {scan_axis: scan_range} + # Get on with the writing now... try: with h5py.File(master_file, "x") as nxsfile: @@ -234,7 +237,6 @@ def eiger_writer( nxsfile, filenames, coordinate_frame, - scan_axis, # This should be omega scan_range, (detector["mode"], n_frames), goniometer, @@ -394,7 +396,7 @@ def write_nxs(**tr_params): def main(): - " Call from the beamline" + "Call from the beamline" # Not the best but it should do the job import argparse from ..command_line import version_parser From a763a9a96e6a36f1cb3c2c88a780883ca1d3ad95 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 30 Mar 2022 18:05:06 +0100 Subject: [PATCH 11/13] Fixed unit test --- tests/test_NxclassWriters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_NxclassWriters.py b/tests/test_NxclassWriters.py index a41db243..ab41061e 100644 --- a/tests/test_NxclassWriters.py +++ b/tests/test_NxclassWriters.py @@ -68,7 +68,7 @@ def test_given_one_data_file_when_write_NXdata_then_data_entry_in_file( {"sam_z": []}, ) assert dummy_nexus_file["/entry/data"].attrs["NX_class"] == b"NXdata" - assert "data" in dummy_nexus_file["/entry/data"] + assert "data_000001" in dummy_nexus_file["/entry/data"] def test_given_scan_axis_when_write_NXdata_then_axis_in_data_entry_with_correct_data_and_attributes( From 6657717347f8bb8333c1bbd01d4474fab188a234 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 30 Mar 2022 18:07:01 +0100 Subject: [PATCH 12/13] Removed no longer used argument --- src/nexgen/beamlines/I03_2d_grid_scan_example.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nexgen/beamlines/I03_2d_grid_scan_example.py b/src/nexgen/beamlines/I03_2d_grid_scan_example.py index ee86c59f..c351e6be 100644 --- a/src/nexgen/beamlines/I03_2d_grid_scan_example.py +++ b/src/nexgen/beamlines/I03_2d_grid_scan_example.py @@ -154,7 +154,6 @@ def example_nexus_file(): source, beam, attenuator, - vds="dataset", metafile=metafile, link_list=dset_links, ) From ffb79170a131b17021d879db43480c7526ca92ae Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Tue, 19 Apr 2022 16:08:10 +0100 Subject: [PATCH 13/13] Added sample depends on for multi axis scan --- src/nexgen/nxs_write/NXclassWriters.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/nexgen/nxs_write/NXclassWriters.py b/src/nexgen/nxs_write/NXclassWriters.py index 4fcec871..30c1d457 100644 --- a/src/nexgen/nxs_write/NXclassWriters.py +++ b/src/nexgen/nxs_write/NXclassWriters.py @@ -189,8 +189,12 @@ def write_NXsample( ), ) else: - # I'm not sure what this should be? - pass + nxsample.create_dataset( + "depends_on", + data=set_dependency( + "phi", path=nxtransformations.name + ), + ) # Create sample_{axisname} groups vectors = split_arrays(coord_frame, goniometer["axes"], goniometer["vectors"])