From a6f2bb265f10a90551b34e252485bd39b7827625 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 19 Feb 2025 08:23:38 -0700 Subject: [PATCH 001/114] IMDReader integration --- .github/actions/setup-deps/action.yaml | 3 + azure-pipelines.yml | 1 + maintainer/conda/environment.yml | 1 + package/MDAnalysis/coordinates/IMD.py | 152 ++++ package/MDAnalysis/coordinates/__init__.py | 1 + package/MDAnalysis/coordinates/base.py | 191 +++++ package/MDAnalysis/coordinates/util.py | 22 + package/doc/sphinx/source/conf.py | 1 + package/pyproject.toml | 1 + .../MDAnalysisTests/coordinates/test_imd.py | 663 ++++++++++++++++++ 10 files changed, 1036 insertions(+) create mode 100644 package/MDAnalysis/coordinates/IMD.py create mode 100644 package/MDAnalysis/coordinates/util.py create mode 100644 testsuite/MDAnalysisTests/coordinates/test_imd.py diff --git a/.github/actions/setup-deps/action.yaml b/.github/actions/setup-deps/action.yaml index 9b4c8042653..87e1f47bb56 100644 --- a/.github/actions/setup-deps/action.yaml +++ b/.github/actions/setup-deps/action.yaml @@ -82,6 +82,8 @@ inputs: default: 'seaborn>=0.7.0' tidynamics: default: 'tidynamics>=1.0.0' + imdclient: + default: 'imdclient' # pip-installed min dependencies coverage: default: 'coverage' @@ -131,6 +133,7 @@ runs: ${{ inputs.distopia }} ${{ inputs.gsd }} ${{ inputs.h5py }} + ${{ inputs.imdclient }} ${{ inputs.hole2 }} ${{ inputs.joblib }} ${{ inputs.netcdf4 }} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 20a32d103ef..38c70fe8cc7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -77,6 +77,7 @@ jobs: displayName: 'Install tools' - script: >- python -m pip install --only-binary=scipy,h5py + imdclient cython hypothesis h5py>=2.10 diff --git a/maintainer/conda/environment.yml b/maintainer/conda/environment.yml index 3ceeeadb2d2..24262552020 100644 --- a/maintainer/conda/environment.yml +++ b/maintainer/conda/environment.yml @@ -31,6 +31,7 @@ dependencies: - sphinxcontrib-bibtex - mdaencore - waterdynamics + - imdclient - pip: - mdahole2 - pathsimanalysis diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py new file mode 100644 index 00000000000..8ddb3daad92 --- /dev/null +++ b/package/MDAnalysis/coordinates/IMD.py @@ -0,0 +1,152 @@ +""" +MDAnalysis IMDReader +^^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: IMDReader + :members: + :inherited-members: + +""" + +from MDAnalysis.coordinates import core +from MDAnalysis.lib.util import store_init_arguments +from MDAnalysis.coordinates.util import parse_host_port +from MDAnalysis.coordinates.base import StreamReaderBase + +try: + import imdclient + from imdclient.IMDClient import IMDClient +except ImportError: + HAS_IMDCLIENT = False + + # Allow building doucmnetation without imdclient + import types + + class MockIMDClient: + pass + imdclient = types.ModuleType("imdclient") + imdclient.IMDClient = MockIMDClient + +else: + HAS_IMDCLIENT = True + +import logging + +logger = logging.getLogger("imdclient.IMDClient") + + +class IMDReader(StreamReaderBase): + """ + Reader for IMD protocol packets. + + Parameters + ---------- + filename : a string of the form "host:port" where host is the hostname + or IP address of the listening GROMACS server and port + is the port number. + n_atoms : int (optional) + number of atoms in the system. defaults to number of atoms + in the topology. don't set this unless you know what you're doing. + kwargs : dict (optional) + keyword arguments passed to the constructed :class:`IMDClient` + """ + + format = "IMD" + one_pass = True + + @store_init_arguments + def __init__( + self, + filename, + convert_units=True, + n_atoms=None, + **kwargs, + ): + if not HAS_IMDCLIENT: + raise ImportError( + "IMDReader requires the imdclient package. " + "Please install it with 'pip install imdclient'." + ) + + super(IMDReader, self).__init__(filename, **kwargs) + + self._imdclient = None + logger.debug("IMDReader initializing") + + if n_atoms is None: + raise ValueError("IMDReader: n_atoms must be specified") + self.n_atoms = n_atoms + + host, port = parse_host_port(filename) + + # This starts the simulation + self._imdclient = IMDClient(host, port, n_atoms, **kwargs) + + imdsinfo = self._imdclient.get_imdsessioninfo() + # NOTE: after testing phase, fail out on IMDv2 + + self.ts = self._Timestep( + self.n_atoms, + positions=imdsinfo.positions, + velocities=imdsinfo.velocities, + forces=imdsinfo.forces, + **self._ts_kwargs, + ) + + self._frame = -1 + + try: + self._read_next_timestep() + except StopIteration: + raise RuntimeError("IMDReader: No data found in stream") + + def _read_frame(self, frame): + + try: + imdf = self._imdclient.get_imdframe() + except EOFError as e: + raise e + + self._frame = frame + self._load_imdframe_into_ts(imdf) + + logger.debug(f"IMDReader: Loaded frame {self._frame}") + return self.ts + + def _load_imdframe_into_ts(self, imdf): + self.ts.frame = self._frame + if imdf.time is not None: + self.ts.time = imdf.time + # NOTE: timestep.pyx "dt" method is suspicious bc it uses "new" keyword for a float + self.ts.data["dt"] = imdf.dt + self.ts.data["step"] = imdf.step + if imdf.energies is not None: + self.ts.data.update( + {k: v for k, v in imdf.energies.items() if k != "step"} + ) + if imdf.box is not None: + self.ts.dimensions = core.triclinic_box(*imdf.box) + if imdf.positions is not None: + # must call copy because reference is expected to reset + # see 'test_frame_collect_all_same' in MDAnalysisTests.coordinates.base + self.ts.positions = imdf.positions + if imdf.velocities is not None: + self.ts.velocities = imdf.velocities + if imdf.forces is not None: + self.ts.forces = imdf.forces + + @staticmethod + def _format_hint(thing): + try: + parse_host_port(thing) + except: + return False + return HAS_IMDCLIENT and True + + def close(self): + """Gracefully shut down the reader. Stops the producer thread.""" + logger.debug("IMDReader close() called") + if self._imdclient is not None: + self._imdclient.stop() + # NOTE: removeme after testing + logger.debug("IMDReader shut down gracefully.") \ No newline at end of file diff --git a/package/MDAnalysis/coordinates/__init__.py b/package/MDAnalysis/coordinates/__init__.py index 602621e5ad3..b9a59af2483 100644 --- a/package/MDAnalysis/coordinates/__init__.py +++ b/package/MDAnalysis/coordinates/__init__.py @@ -770,6 +770,7 @@ class can choose an appropriate reader automatically. from . import DMS from . import GMS from . import GRO +from . import IMD from . import INPCRD from . import LAMMPS from . import MOL2 diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 61afa29e7da..09be0dc1e34 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1841,3 +1841,194 @@ def __repr__(self): def convert(self, obj): raise NotImplementedError + +class StreamReaderBase(ReaderBase): + + def __init__(self, filename, convert_units=True, **kwargs): + super(StreamReaderBase, self).__init__( + filename, convert_units=convert_units, **kwargs + ) + self._init_scope = True + self._reopen_called = False + self._first_ts = None + + def _read_next_timestep(self): + # No rewinding- to both load the first frame after __init__ + # and access it again during iteration, we need to store first ts in mem + if not self._init_scope and self._frame == -1: + self._frame += 1 + # can't simply return the same ts again- transformations would be applied twice + # instead, return the pre-transformed copy + return self._first_ts + + ts = self._read_frame(self._frame + 1) + + if self._init_scope: + self._first_ts = self.ts.copy() + self._init_scope = False + + return ts + + @property + def n_frames(self): + """Changes as stream is processed unlike other readers""" + raise RuntimeError( + "{}: n_frames is unknown".format(self.__class__.__name__) + ) + + def __len__(self): + raise RuntimeError( + "{} has unknown length".format(self.__class__.__name__) + ) + + def next(self): + """Don't rewind after iteration. When _reopen() is called, + an error will be raised + """ + try: + ts = self._read_next_timestep() + except (EOFError, IOError): + # Don't rewind here like we normally would + raise StopIteration from None + else: + for auxname, reader in self._auxs.items(): + ts = self._auxs[auxname].update_ts(ts) + + ts = self._apply_transformations(ts) + + return ts + + def rewind(self): + """Raise error on rewind""" + raise RuntimeError( + "{}: Stream-based readers can't be rewound".format( + self.__class__.__name__ + ) + ) + + # Incompatible methods + def copy(self): + raise NotImplementedError( + "{} does not support copying".format(self.__class__.__name__) + ) + + def _reopen(self): + if self._reopen_called: + raise RuntimeError( + "{}: Cannot reopen stream".format(self.__class__.__name__) + ) + self._frame = -1 + self._reopen_called = True + + def __getitem__(self, frame): + """Return the Timestep corresponding to *frame*. + + If `frame` is a integer then the corresponding frame is + returned. Negative numbers are counted from the end. + + If frame is a :class:`slice` then an iterator is returned that + allows iteration over that part of the trajectory. + + Note + ---- + *frame* is a 0-based frame index. + """ + if isinstance(frame, slice): + _, _, step = self.check_slice_indices( + frame.start, frame.stop, frame.step + ) + if step is None: + return FrameIteratorAll(self) + else: + return StreamFrameIteratorSliced(self, step) + else: + raise TypeError( + "Streamed trajectories must be an indexed using a slice" + ) + + def check_slice_indices(self, start, stop, step): + if start is not None: + raise ValueError( + "{}: Cannot expect a start index from a stream, 'start' must be None".format( + self.__class__.__name__ + ) + ) + if stop is not None: + raise ValueError( + "{}: Cannot expect a stop index from a stream, 'stop' must be None".format( + self.__class__.__name__ + ) + ) + if step is not None: + if isinstance(step, numbers.Integral): + if step < 1: + raise ValueError( + "{}: Cannot go backwards in a stream, 'step' must be > 0".format( + self.__class__.__name__ + ) + ) + else: + raise ValueError( + "{}: 'step' must be an integer".format( + self.__class__.__name__ + ) + ) + + return start, stop, step + + def __getstate__(self): + raise NotImplementedError( + "{} does not support pickling".format(self.__class__.__name__) + ) + + def __setstate__(self, state: object): + raise NotImplementedError( + "{} does not support pickling".format(self.__class__.__name__) + ) + + def __repr__(self): + return ( + "<{cls} {fname} with continuous stream of {natoms} atoms>" + "".format( + cls=self.__class__.__name__, + fname=self.filename, + natoms=self.n_atoms, + ) + ) + + +class StreamFrameIteratorSliced(FrameIteratorBase): + + def __init__(self, trajectory, step): + super().__init__(trajectory) + self._step = step + + def __iter__(self): + # Calling reopen tells reader + # it can't be reopened again + self.trajectory._reopen() + return self + + def __next__(self): + try: + # Burn the timesteps until we reach the desired step + # Don't use next() to avoid unnecessary transformations + while self.trajectory._frame + 1 % self.step != 0: + self.trajectory._read_next_timestep() + except (EOFError, IOError): + # Don't rewind here like we normally would + raise StopIteration from None + + return self.trajectory.next() + + def __len__(self): + raise RuntimeError( + "{} has unknown length".format(self.__class__.__name__) + ) + + def __getitem__(self, frame): + raise RuntimeError("Sliced iterator does not support indexing") + + @property + def step(self): + return self._step \ No newline at end of file diff --git a/package/MDAnalysis/coordinates/util.py b/package/MDAnalysis/coordinates/util.py new file mode 100644 index 00000000000..7b506c513e6 --- /dev/null +++ b/package/MDAnalysis/coordinates/util.py @@ -0,0 +1,22 @@ +import logging +import select +import time + +logger = logging.getLogger("imdclient.IMDClient") + +# NOTE: think of other edge cases as well- should be robust +def parse_host_port(filename): + if not filename.startswith("imd://"): + raise ValueError("IMDReader: URL must be in the format 'imd://host:port'") + + # Check if the format is correct + parts = filename.split("imd://")[1].split(":") + if len(parts) == 2: + host = parts[0] + try: + port = int(parts[1]) + return (host, port) + except ValueError: + raise ValueError("IMDReader: Port must be an integer") + else: + raise ValueError("IMDReader: URL must be in the format 'imd://host:port'") diff --git a/package/doc/sphinx/source/conf.py b/package/doc/sphinx/source/conf.py index d1a288ed346..9f24a98886d 100644 --- a/package/doc/sphinx/source/conf.py +++ b/package/doc/sphinx/source/conf.py @@ -349,4 +349,5 @@ class KeyStyle(UnsrtStyle): "pathsimanalysis": ("https://www.mdanalysis.org/PathSimAnalysis/", None), "mdahole2": ("https://www.mdanalysis.org/mdahole2/", None), "dask": ("https://docs.dask.org/en/stable/", None), + "imdclient": ("https://imdclient.readthedocs.io/en/stable/", None), } diff --git a/package/pyproject.toml b/package/pyproject.toml index 8d0e6a86bcd..605c1396826 100644 --- a/package/pyproject.toml +++ b/package/pyproject.toml @@ -79,6 +79,7 @@ extra_formats = [ "pytng>=0.2.3", "gsd>3.0.0", "rdkit>=2020.03.1", + "imdclient", ] analysis = [ "biopython>=1.80", diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py new file mode 100644 index 00000000000..b04cbf53111 --- /dev/null +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -0,0 +1,663 @@ +"""Test for MDAnalysis trajectory reader expectations +""" +import numpy as np +import logging +import pytest +from MDAnalysis.transformations import translate +import pickle +import MDAnalysis as mda +from numpy.testing import ( + assert_almost_equal, + assert_array_almost_equal, + assert_equal, + assert_allclose, +) +from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + +if HAS_IMDCLIENT: + import imdclient + from imdclient.tests.utils import ( + get_free_port, + create_default_imdsinfo_v3, + ) + from imdclient.tests.server import InThreadIMDServer + +from MDAanalysis.coordinates.IMD import IMDReader + +from MDAnalysisTests.datafiles import ( + COORDINATES_TOPOLOGY, + COORDINATES_TRR, + COORDINATES_H5MD, +) + +from MDAnalysisTests.coordinates.base import ( + MultiframeReaderTest, + BaseReference, + assert_timestep_almost_equal, +) + +logger = logging.getLogger("imdclient.IMDClient") +file_handler = logging.FileHandler("test.log") +formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +file_handler.setFormatter(formatter) +logger.addHandler(file_handler) +logger.setLevel(logging.DEBUG) + +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not isntalled") +class IMDReference(BaseReference): + def __init__(self): + super(IMDReference, self).__init__() + self.port = get_free_port() + # Serve TRR traj data via the server + traj = mda.coordinates.TRR.TRRReader(COORDINATES_TRR) + self.server = InThreadIMDServer(traj) + self.server.set_imdsessioninfo(create_default_imdsinfo_v3()) + + self.n_atoms = traj.n_atoms + self.prec = 3 + + self.trajectory = f"imd://localhost:{self.port}" + self.topology = COORDINATES_TOPOLOGY + self.changing_dimensions = True + self.reader = IMDReader + + self.first_frame.velocities = self.first_frame.positions / 10 + self.first_frame.forces = self.first_frame.positions / 100 + + self.second_frame.velocities = self.second_frame.positions / 10 + self.second_frame.forces = self.second_frame.positions / 100 + + self.last_frame.velocities = self.last_frame.positions / 10 + self.last_frame.forces = self.last_frame.positions / 100 + + self.jump_to_frame.velocities = self.jump_to_frame.positions / 10 + self.jump_to_frame.forces = self.jump_to_frame.positions / 100 + + def iter_ts(self, i): + ts = self.first_frame.copy() + ts.positions = 2**i * self.first_frame.positions + ts.velocities = ts.positions / 10 + ts.forces = ts.positions / 100 + ts.time = i + ts.frame = i + return ts + + +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not isntalled") +class TestIMDReaderBaseAPI(MultiframeReaderTest): + + @pytest.fixture() + def ref(self): + """Not a static method like in base class- need new server for each test""" + return IMDReference() + + @pytest.fixture() + def reader(self, ref): + # This will start the test IMD Server, waiting for a connection + # to then send handshake & first frame + ref.server.handshake_sequence("localhost", ref.port) + # This will connect to the test IMD Server and read the first frame + reader = ref.reader(ref.trajectory, n_atoms=ref.n_atoms) + # Send the rest of the frames- small enough to all fit in socket itself + ref.server.send_frames(1, 5) + + reader.add_auxiliary( + "lowf", + ref.aux_lowf, + dt=ref.aux_lowf_dt, + initial_time=0, + time_selector=None, + ) + reader.add_auxiliary( + "highf", + ref.aux_highf, + dt=ref.aux_highf_dt, + initial_time=0, + time_selector=None, + ) + return reader + + @pytest.fixture() + def transformed(self, ref): + # This will start the test IMD Server, waiting for a connection + # to then send handshake & first frame + ref.server.handshake_sequence("localhost", ref.port) + # This will connect to the test IMD Server and read the first frame + transformed = ref.reader(ref.trajectory, n_atoms=ref.n_atoms) + # Send the rest of the frames- small enough to all fit in socket itself + ref.server.send_frames(1, 5) + transformed.add_transformations( + translate([1, 1, 1]), translate([0, 0, 0.33]) + ) + return transformed + + @pytest.mark.skip( + reason="Stream-based reader cannot determine n_frames until EOF" + ) + def test_n_frames(self, reader, ref): + assert_equal( + self.universe.trajectory.n_frames, + 1, + "wrong number of frames in pdb", + ) + + def test_first_frame(self, ref, reader): + # don't rewind here as in inherited base test + assert_timestep_almost_equal( + reader.ts, ref.first_frame, decimal=ref.prec + ) + + @pytest.mark.skip(reason="IMD is not a writeable format") + def test_get_writer_1(self, ref, reader, tmpdir): + with tmpdir.as_cwd(): + outfile = "test-writer." + ref.ext + with reader.Writer(outfile) as W: + assert_equal(isinstance(W, ref.writer), True) + assert_equal(W.n_atoms, reader.n_atoms) + + @pytest.mark.skip(reason="IMD is not a writeable format") + def test_get_writer_2(self, ref, reader, tmpdir): + with tmpdir.as_cwd(): + outfile = "test-writer." + ref.ext + with reader.Writer(outfile, n_atoms=100) as W: + assert_equal(isinstance(W, ref.writer), True) + assert_equal(W.n_atoms, 100) + + @pytest.mark.skip( + reason="Stream-based reader cannot determine total_time until EOF" + ) + def test_total_time(self, reader, ref): + assert_almost_equal( + reader.totaltime, + ref.totaltime, + decimal=ref.prec, + ) + + @pytest.mark.skip(reason="Stream-based reader can only be read iteratively") + def test_changing_dimensions(self, ref, reader): + if ref.changing_dimensions: + reader.rewind() + if ref.dimensions is None: + assert reader.ts.dimensions is None + else: + assert_array_almost_equal( + reader.ts.dimensions, + ref.dimensions, + decimal=ref.prec, + ) + reader[1] + if ref.dimensions_second_frame is None: + assert reader.ts.dimensions is None + else: + assert_array_almost_equal( + reader.ts.dimensions, + ref.dimensions_second_frame, + decimal=ref.prec, + ) + + def test_iter(self, ref, reader): + for i, ts in enumerate(reader): + assert_timestep_almost_equal(ts, ref.iter_ts(i), decimal=ref.prec) + + def test_first_dimensions(self, ref, reader): + # don't rewind here as in inherited base test + if ref.dimensions is None: + assert reader.ts.dimensions is None + else: + assert_array_almost_equal( + reader.ts.dimensions, + ref.dimensions, + decimal=ref.prec, + ) + + def test_volume(self, ref, reader): + # don't rewind here as in inherited base test + vol = reader.ts.volume + # Here we can only be sure about the numbers upto the decimal point due + # to floating point impressions. + assert_almost_equal(vol, ref.volume, 0) + + @pytest.mark.skip(reason="Cannot create new reader from same stream") + def test_reload_auxiliaries_from_description(self, ref, reader): + # get auxiliary desscriptions form existing reader + descriptions = reader.get_aux_descriptions() + # load a new reader, without auxiliaries + reader = ref.reader(ref.trajectory) + # load auxiliaries into new reader, using description... + for aux in descriptions: + reader.add_auxiliary(**aux) + # should have the same number of auxiliaries + assert_equal( + reader.aux_list, + reader.aux_list, + "Number of auxiliaries does not match", + ) + # each auxiliary should be the same + for auxname in reader.aux_list: + assert_equal( + reader._auxs[auxname], + reader._auxs[auxname], + "AuxReaders do not match", + ) + + @pytest.mark.skip(reason="Stream can only be read in for loop") + def test_stop_iter(self, reader): + # reset to 0 + reader.rewind() + for ts in reader[:-1]: + pass + assert_equal(reader.frame, 0) + + @pytest.mark.skip(reason="Cannot rewind stream") + def test_iter_rewinds(self, reader, accessor): + for ts_indices in accessor(reader): + pass + assert_equal(ts_indices.frame, 0) + + @pytest.mark.skip( + reason="Timeseries currently requires n_frames to be known" + ) + @pytest.mark.parametrize( + "order", ("fac", "fca", "afc", "acf", "caf", "cfa") + ) + def test_timeseries_shape(self, reader, order): + timeseries = reader.timeseries(order=order) + a_index = order.index("a") + # f_index = order.index("f") + c_index = order.index("c") + assert timeseries.shape[a_index] == reader.n_atoms + # assert timeseries.shape[f_index] == len(reader) + assert timeseries.shape[c_index] == 3 + + @pytest.mark.skip( + reason="Timeseries currently requires n_frames to be known" + ) + @pytest.mark.parametrize("asel", ("index 1", "index 2", "index 1 to 3")) + def test_timeseries_asel_shape(self, reader, asel): + atoms = mda.Universe(reader.filename).select_atoms(asel) + timeseries = reader.timeseries(atoms, order="fac") + assert timeseries.shape[0] == len(reader) + assert timeseries.shape[1] == len(atoms) + assert timeseries.shape[2] == 3 + + @pytest.mark.skip("Cannot slice stream") + @pytest.mark.parametrize("slice", ([0, 2, 1], [0, 10, 2], [0, 10, 3])) + def test_timeseries_values(self, reader, slice): + ts_positions = [] + if isinstance(reader, mda.coordinates.memory.MemoryReader): + pytest.xfail( + "MemoryReader uses deprecated stop inclusive" + " indexing, see Issue #3893" + ) + if slice[1] > len(reader): + pytest.skip("too few frames in reader") + for i in range(slice[0], slice[1], slice[2]): + ts = reader[i] + ts_positions.append(ts.positions.copy()) + positions = np.asarray(ts_positions) + timeseries = reader.timeseries( + start=slice[0], + stop=slice[1], + step=slice[2], + order="fac", + ) + assert_allclose(timeseries, positions) + + @pytest.mark.skip(reason="Cannot rewind stream") + def test_transformations_2iter(self, ref, transformed): + # Are the transformations applied and + # are the coordinates "overtransformed"? + v1 = np.float32((1, 1, 1)) + v2 = np.float32((0, 0, 0.33)) + idealcoords = [] + for i, ts in enumerate(transformed): + idealcoords.append(ref.iter_ts(i).positions + v1 + v2) + assert_array_almost_equal( + ts.positions, + idealcoords[i], + decimal=ref.prec, + ) + + for i, ts in enumerate(transformed): + assert_almost_equal( + ts.positions, + idealcoords[i], + decimal=ref.prec, + ) + + @pytest.mark.skip(reason="Cannot slice stream") + def test_transformations_slice(self, ref, transformed): + # Are the transformations applied when iterating over a slice of the trajectory? + v1 = np.float32((1, 1, 1)) + v2 = np.float32((0, 0, 0.33)) + for i, ts in enumerate(transformed[2:3:1]): + idealcoords = ref.iter_ts(ts.frame).positions + v1 + v2 + assert_array_almost_equal( + ts.positions, idealcoords, decimal=ref.prec + ) + + @pytest.mark.skip(reason="Cannot slice stream") + def test_transformations_switch_frame(self, ref, transformed): + # This test checks if the transformations are applied and if the coordinates + # "overtransformed" on different situations + # Are the transformations applied when we switch to a different frame? + v1 = np.float32((1, 1, 1)) + v2 = np.float32((0, 0, 0.33)) + first_ideal = ref.iter_ts(0).positions + v1 + v2 + if len(transformed) > 1: + assert_array_almost_equal( + transformed[0].positions, + first_ideal, + decimal=ref.prec, + ) + second_ideal = ref.iter_ts(1).positions + v1 + v2 + assert_array_almost_equal( + transformed[1].positions, + second_ideal, + decimal=ref.prec, + ) + + # What if we comeback to the previous frame? + assert_array_almost_equal( + transformed[0].positions, + first_ideal, + decimal=ref.prec, + ) + + # How about we switch the frame to itself? + assert_array_almost_equal( + transformed[0].positions, + first_ideal, + decimal=ref.prec, + ) + else: + assert_array_almost_equal( + transformed[0].positions, + first_ideal, + decimal=ref.prec, + ) + + @pytest.mark.skip(reason="Cannot rewind stream") + def test_transformation_rewind(self, ref, transformed): + # this test checks if the transformations are applied after rewinding the + # trajectory + v1 = np.float32((1, 1, 1)) + v2 = np.float32((0, 0, 0.33)) + ideal_coords = ref.iter_ts(0).positions + v1 + v2 + transformed.rewind() + assert_array_almost_equal( + transformed[0].positions, + ideal_coords, + decimal=ref.prec, + ) + + @pytest.mark.skip(reason="Cannot make a copy of a stream") + def test_copy(self, ref, transformed): + # this test checks if transformations are carried over a copy and if the + # coordinates of the copy are equal to the original's + v1 = np.float32((1, 1, 1)) + v2 = np.float32((0, 0, 0.33)) + new = transformed.copy() + assert_equal( + transformed.transformations, + new.transformations, + "transformations are not equal", + ) + for i, ts in enumerate(new): + ideal_coords = ref.iter_ts(i).positions + v1 + v2 + assert_array_almost_equal( + ts.positions, ideal_coords, decimal=ref.prec + ) + + @pytest.mark.skip(reason="Cannot pickle socket") + def test_pickle_reader(self, reader): + """It probably wouldn't be a good idea to pickle a + reader that is connected to a server""" + reader_p = pickle.loads(pickle.dumps(reader)) + assert_equal(len(reader), len(reader_p)) + assert_equal( + reader.ts, + reader_p.ts, + "Timestep is changed after pickling", + ) + + @pytest.mark.skip(reason="Cannot pickle socket") + def test_pickle_next_ts_reader(self, reader): + reader_p = pickle.loads(pickle.dumps(reader)) + assert_equal( + next(reader), + next(reader_p), + "Next timestep is changed after pickling", + ) + + @pytest.mark.skip(reason="Cannot pickle socket") + def test_pickle_last_ts_reader(self, reader): + # move current ts to last frame. + reader[-1] + reader_p = pickle.loads(pickle.dumps(reader)) + assert_equal( + len(reader), + len(reader_p), + "Last timestep is changed after pickling", + ) + assert_equal( + reader.ts, + reader_p.ts, + "Last timestep is changed after pickling", + ) + + @pytest.mark.skip(reason="Cannot copy stream") + def test_transformations_copy(self, ref, transformed): + # this test checks if transformations are carried over a copy and if the + # coordinates of the copy are equal to the original's + v1 = np.float32((1, 1, 1)) + v2 = np.float32((0, 0, 0.33)) + new = transformed.copy() + assert_equal( + transformed.transformations, + new.transformations, + "transformations are not equal", + ) + for i, ts in enumerate(new): + ideal_coords = ref.iter_ts(i).positions + v1 + v2 + assert_array_almost_equal( + ts.positions, ideal_coords, decimal=ref.prec + ) + + @pytest.mark.skip( + reason="Timeseries currently requires n_frames to be known" + ) + def test_timeseries_empty_asel(self, reader): + with pytest.warns( + UserWarning, + match="Empty string to select atoms, empty group returned.", + ): + atoms = mda.Universe(reader.filename).select_atoms(None) + with pytest.raises(ValueError, match="Timeseries requires at least"): + reader.timeseries(asel=atoms) + + @pytest.mark.skip( + reason="Timeseries currently requires n_frames to be known" + ) + def test_timeseries_empty_atomgroup(self, reader): + with pytest.warns( + UserWarning, + match="Empty string to select atoms, empty group returned.", + ): + atoms = mda.Universe(reader.filename).select_atoms(None) + with pytest.raises(ValueError, match="Timeseries requires at least"): + reader.timeseries(atomgroup=atoms) + + @pytest.mark.skip( + reason="Timeseries currently requires n_frames to be known" + ) + def test_timeseries_asel_warns_deprecation(self, reader): + atoms = mda.Universe(reader.filename).select_atoms("index 1") + with pytest.warns(DeprecationWarning, match="asel argument to"): + timeseries = reader.timeseries(asel=atoms, order="fac") + + @pytest.mark.skip( + reason="Timeseries currently requires n_frames to be known" + ) + def test_timeseries_atomgroup(self, reader): + atoms = mda.Universe(reader.filename).select_atoms("index 1") + timeseries = reader.timeseries(atomgroup=atoms, order="fac") + + @pytest.mark.skip( + reason="Timeseries currently requires n_frames to be known" + ) + def test_timeseries_atomgroup_asel_mutex(self, reader): + atoms = mda.Universe(reader.filename).select_atoms("index 1") + with pytest.raises(ValueError, match="Cannot provide both"): + timeseries = reader.timeseries( + atomgroup=atoms, asel=atoms, order="fac" + ) + + @pytest.mark.skip("Cannot slice stream") + def test_last_frame(self, ref, reader): + ts = reader[-1] + assert_timestep_almost_equal(ts, ref.last_frame, decimal=ref.prec) + + @pytest.mark.skip("Cannot slice stream") + def test_go_over_last_frame(self, ref, reader): + with pytest.raises(IndexError): + reader[ref.n_frames + 1] + + @pytest.mark.skip("Cannot slice stream") + def test_frame_jump(self, ref, reader): + ts = reader[ref.jump_to_frame.frame] + assert_timestep_almost_equal(ts, ref.jump_to_frame, decimal=ref.prec) + + @pytest.mark.skip("Cannot slice stream") + def test_frame_jump_issue1942(self, ref, reader): + """Test for issue 1942 (especially XDR on macOS)""" + reader.rewind() + try: + for ii in range(ref.n_frames + 2): + reader[0] + except StopIteration: + pytest.fail("Frame-seeking wrongly iterated (#1942)") + + def test_next_gives_second_frame(self, ref, reader): + # don't recreate reader here as in inherited base test + ts = reader.next() + assert_timestep_almost_equal(ts, ref.second_frame, decimal=ref.prec) + + @pytest.mark.skip( + reason="Stream isn't rewound after iteration- base reference is the same but it is the last frame" + ) + def test_frame_collect_all_same(self, reader): + # check that the timestep resets so that the base reference is the same + # for all timesteps in a collection with the exception of memoryreader + # and DCDReader + if isinstance(reader, mda.coordinates.memory.MemoryReader): + pytest.xfail("memoryreader allows independent coordinates") + if isinstance(reader, mda.coordinates.DCD.DCDReader): + pytest.xfail( + "DCDReader allows independent coordinates." + "This behaviour is deprecated and will be changed" + "in 3.0" + ) + collected_ts = [] + for i, ts in enumerate(reader): + collected_ts.append(ts.positions) + for array in collected_ts: + assert_allclose(array, collected_ts[0]) + + +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not isntalled") +class TestStreamIteration: + + @pytest.fixture + def port(self): + return get_free_port() + + @pytest.fixture + def universe(self): + return mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) + + @pytest.fixture + def imdsinfo(self): + return create_default_imdsinfo_v3() + + @pytest.fixture + def reader(self, universe, imdsinfo, port): + server = InThreadIMDServer(universe.trajectory) + server.set_imdsessioninfo(imdsinfo) + server.handshake_sequence("localhost", port, first_frame=True) + reader = IMDReader( + f"imd://localhost:{port}", + n_atoms=universe.trajectory.n_atoms, + ) + server.send_frames(1, 5) + + yield reader + server.cleanup() + + def test_iterate_step(self, reader, universe): + i = 0 + for ts in reader[::2]: + assert ts.frame == i + i += 2 + + def test_iterate_twice_sliced_raises_error(self, reader): + for ts in reader[::2]: + pass + with pytest.raises(RuntimeError): + for ts in reader[::2]: + pass + + def test_iterate_twice_all_raises_error(self, reader): + for ts in reader: + pass + with pytest.raises(RuntimeError): + for ts in reader: + pass + + def test_iterate_twice_fi_all_raises_error(self, reader): + for ts in reader[:]: + pass + with pytest.raises(RuntimeError): + for ts in reader[:]: + pass + + def test_index_stream_raises_error(self, reader): + with pytest.raises(TypeError): + reader[0] + + def test_iterate_backwards_raises_error(self, reader): + with pytest.raises(ValueError): + for ts in reader[::-1]: + pass + + def test_iterate_start_stop_raises_error(self, reader): + with pytest.raises(ValueError): + for ts in reader[1:3]: + pass + + def test_subslice_fi_all_after_iteration_raises_error(self, reader): + sliced_reader = reader[:] + for ts in sliced_reader: + pass + sub_sliced_reader = sliced_reader[::1] + with pytest.raises(RuntimeError): + for ts in sub_sliced_reader: + pass + + +def test_n_atoms_mismatch(): + universe = mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) + port = get_free_port() + server = InThreadIMDServer(universe.trajectory) + server.set_imdsessioninfo(create_default_imdsinfo_v3()) + server.handshake_sequence("localhost", port, first_frame=True) + with pytest.raises( + EOFError, + match="IMDProducer: Expected n_atoms value 6, got 5. Ensure you are using the correct topology file.", + ): + IMDReader( + f"imd://localhost:{port}", + n_atoms=universe.trajectory.n_atoms + 1, + ) \ No newline at end of file From a67cbfb0df0a2974039e53e2c94f4aa20790dc6d Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 19 Feb 2025 15:32:39 -0700 Subject: [PATCH 002/114] small cleanup --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index b04cbf53111..75f25eff3f0 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -22,7 +22,7 @@ ) from imdclient.tests.server import InThreadIMDServer -from MDAanalysis.coordinates.IMD import IMDReader +from MDAnalysis.coordinates.IMD import IMDReader from MDAnalysisTests.datafiles import ( COORDINATES_TOPOLOGY, @@ -646,7 +646,7 @@ def test_subslice_fi_all_after_iteration_raises_error(self, reader): for ts in sub_sliced_reader: pass - +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") def test_n_atoms_mismatch(): universe = mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) port = get_free_port() From 8c00bc12f5407ec7c380acc0b5d52329e2a19afb Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 9 Apr 2025 17:12:56 -0700 Subject: [PATCH 003/114] Chore: Cleanup and small fixes 1. Moved `parse_host_port` to `IMD.py` and deleted `util.py` 2. Cleaned up `test_imd.py` - changes to `assert_*` functions and simplified non-applicable test to pass automatically --- package/MDAnalysis/coordinates/IMD.py | 44 +- package/MDAnalysis/coordinates/util.py | 22 - .../MDAnalysisTests/coordinates/test_imd.py | 403 ++---------------- 3 files changed, 73 insertions(+), 396 deletions(-) delete mode 100644 package/MDAnalysis/coordinates/util.py diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 8ddb3daad92..9675e5bcd0d 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -8,9 +8,11 @@ """ +import numpy as np +import logging + from MDAnalysis.coordinates import core from MDAnalysis.lib.util import store_init_arguments -from MDAnalysis.coordinates.util import parse_host_port from MDAnalysis.coordinates.base import StreamReaderBase try: @@ -19,7 +21,7 @@ except ImportError: HAS_IMDCLIENT = False - # Allow building doucmnetation without imdclient + # Allow building documentation without imdclient import types class MockIMDClient: @@ -30,9 +32,7 @@ class MockIMDClient: else: HAS_IMDCLIENT = True -import logging - -logger = logging.getLogger("imdclient.IMDClient") +logger = logging.getLogger("MDAnalysis.coordinates.IMDReader") class IMDReader(StreamReaderBase): @@ -41,12 +41,12 @@ class IMDReader(StreamReaderBase): Parameters ---------- - filename : a string of the form "host:port" where host is the hostname + filename : a string of the form "imd://host:port" where host is the hostname or IP address of the listening GROMACS server and port is the port number. n_atoms : int (optional) number of atoms in the system. defaults to number of atoms - in the topology. don't set this unless you know what you're doing. + in the topology. Don't set this unless you know what you're doing. kwargs : dict (optional) keyword arguments passed to the constructed :class:`IMDClient` """ @@ -97,8 +97,8 @@ def __init__( try: self._read_next_timestep() - except StopIteration: - raise RuntimeError("IMDReader: No data found in stream") + except StopIteration as e: + raise RuntimeError("IMDReader: No data found in stream") from e def _read_frame(self, frame): @@ -110,7 +110,7 @@ def _read_frame(self, frame): self._frame = frame self._load_imdframe_into_ts(imdf) - logger.debug(f"IMDReader: Loaded frame {self._frame}") + logger.debug("IMDReader: Loaded frame %d", self._frame) return self.ts def _load_imdframe_into_ts(self, imdf): @@ -129,11 +129,11 @@ def _load_imdframe_into_ts(self, imdf): if imdf.positions is not None: # must call copy because reference is expected to reset # see 'test_frame_collect_all_same' in MDAnalysisTests.coordinates.base - self.ts.positions = imdf.positions + np.copyto(self.ts.positions, imdf.positions) if imdf.velocities is not None: - self.ts.velocities = imdf.velocities + np.copyto(self.ts.velocities, imdf.velocities) if imdf.forces is not None: - self.ts.forces = imdf.forces + np.copyto(self.ts.forces, imdf.forces) @staticmethod def _format_hint(thing): @@ -149,4 +149,20 @@ def close(self): if self._imdclient is not None: self._imdclient.stop() # NOTE: removeme after testing - logger.debug("IMDReader shut down gracefully.") \ No newline at end of file + logger.debug("IMDReader shut down gracefully.") + +# NOTE: think of other edge cases as well- should be robust +def parse_host_port(filename): + if not filename.startswith("imd://"): + raise ValueError("IMDReader: URL must be in the format 'imd://host:port'") + # Check if the format is correct + parts = filename.split("imd://")[1].split(":") + if len(parts) == 2: + host = parts[0] + try: + port = int(parts[1]) + return (host, port) + except ValueError as e: + raise ValueError("IMDReader: Port must be an integer") from e + else: + raise ValueError("IMDReader: URL must be in the format 'imd://host:port'") diff --git a/package/MDAnalysis/coordinates/util.py b/package/MDAnalysis/coordinates/util.py deleted file mode 100644 index 7b506c513e6..00000000000 --- a/package/MDAnalysis/coordinates/util.py +++ /dev/null @@ -1,22 +0,0 @@ -import logging -import select -import time - -logger = logging.getLogger("imdclient.IMDClient") - -# NOTE: think of other edge cases as well- should be robust -def parse_host_port(filename): - if not filename.startswith("imd://"): - raise ValueError("IMDReader: URL must be in the format 'imd://host:port'") - - # Check if the format is correct - parts = filename.split("imd://")[1].split(":") - if len(parts) == 2: - host = parts[0] - try: - port = int(parts[1]) - return (host, port) - except ValueError: - raise ValueError("IMDReader: Port must be an integer") - else: - raise ValueError("IMDReader: URL must be in the format 'imd://host:port'") diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 75f25eff3f0..753453e023c 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -36,16 +36,7 @@ assert_timestep_almost_equal, ) -logger = logging.getLogger("imdclient.IMDClient") -file_handler = logging.FileHandler("test.log") -formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s" -) -file_handler.setFormatter(formatter) -logger.addHandler(file_handler) -logger.setLevel(logging.DEBUG) - -@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not isntalled") +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class IMDReference(BaseReference): def __init__(self): super(IMDReference, self).__init__() @@ -85,10 +76,10 @@ def iter_ts(self, i): return ts -@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not isntalled") +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class TestIMDReaderBaseAPI(MultiframeReaderTest): - @pytest.fixture() + @pytest.fixture(scope='function') def ref(self): """Not a static method like in base class- need new server for each test""" return IMDReference() @@ -133,15 +124,8 @@ def transformed(self, ref): ) return transformed - @pytest.mark.skip( - reason="Stream-based reader cannot determine n_frames until EOF" - ) def test_n_frames(self, reader, ref): - assert_equal( - self.universe.trajectory.n_frames, - 1, - "wrong number of frames in pdb", - ) + pass def test_first_frame(self, ref, reader): # don't rewind here as in inherited base test @@ -149,53 +133,17 @@ def test_first_frame(self, ref, reader): reader.ts, ref.first_frame, decimal=ref.prec ) - @pytest.mark.skip(reason="IMD is not a writeable format") def test_get_writer_1(self, ref, reader, tmpdir): - with tmpdir.as_cwd(): - outfile = "test-writer." + ref.ext - with reader.Writer(outfile) as W: - assert_equal(isinstance(W, ref.writer), True) - assert_equal(W.n_atoms, reader.n_atoms) + pass - @pytest.mark.skip(reason="IMD is not a writeable format") def test_get_writer_2(self, ref, reader, tmpdir): - with tmpdir.as_cwd(): - outfile = "test-writer." + ref.ext - with reader.Writer(outfile, n_atoms=100) as W: - assert_equal(isinstance(W, ref.writer), True) - assert_equal(W.n_atoms, 100) - - @pytest.mark.skip( - reason="Stream-based reader cannot determine total_time until EOF" - ) + pass + def test_total_time(self, reader, ref): - assert_almost_equal( - reader.totaltime, - ref.totaltime, - decimal=ref.prec, - ) + pass - @pytest.mark.skip(reason="Stream-based reader can only be read iteratively") def test_changing_dimensions(self, ref, reader): - if ref.changing_dimensions: - reader.rewind() - if ref.dimensions is None: - assert reader.ts.dimensions is None - else: - assert_array_almost_equal( - reader.ts.dimensions, - ref.dimensions, - decimal=ref.prec, - ) - reader[1] - if ref.dimensions_second_frame is None: - assert reader.ts.dimensions is None - else: - assert_array_almost_equal( - reader.ts.dimensions, - ref.dimensions_second_frame, - decimal=ref.prec, - ) + pass def test_iter(self, ref, reader): for i, ts in enumerate(reader): @@ -206,365 +154,100 @@ def test_first_dimensions(self, ref, reader): if ref.dimensions is None: assert reader.ts.dimensions is None else: - assert_array_almost_equal( + assert_allclose( reader.ts.dimensions, ref.dimensions, - decimal=ref.prec, + rtol=0, + atol=1.5 * 10**(-ref.prec) ) + def test_volume(self, ref, reader): # don't rewind here as in inherited base test vol = reader.ts.volume # Here we can only be sure about the numbers upto the decimal point due # to floating point impressions. - assert_almost_equal(vol, ref.volume, 0) + assert_allclose(vol, ref.volume, rtol=0, atol=1.5e0) - @pytest.mark.skip(reason="Cannot create new reader from same stream") def test_reload_auxiliaries_from_description(self, ref, reader): - # get auxiliary desscriptions form existing reader - descriptions = reader.get_aux_descriptions() - # load a new reader, without auxiliaries - reader = ref.reader(ref.trajectory) - # load auxiliaries into new reader, using description... - for aux in descriptions: - reader.add_auxiliary(**aux) - # should have the same number of auxiliaries - assert_equal( - reader.aux_list, - reader.aux_list, - "Number of auxiliaries does not match", - ) - # each auxiliary should be the same - for auxname in reader.aux_list: - assert_equal( - reader._auxs[auxname], - reader._auxs[auxname], - "AuxReaders do not match", - ) + pass - @pytest.mark.skip(reason="Stream can only be read in for loop") def test_stop_iter(self, reader): - # reset to 0 - reader.rewind() - for ts in reader[:-1]: - pass - assert_equal(reader.frame, 0) + pass - @pytest.mark.skip(reason="Cannot rewind stream") def test_iter_rewinds(self, reader, accessor): - for ts_indices in accessor(reader): - pass - assert_equal(ts_indices.frame, 0) + pass - @pytest.mark.skip( - reason="Timeseries currently requires n_frames to be known" - ) - @pytest.mark.parametrize( - "order", ("fac", "fca", "afc", "acf", "caf", "cfa") - ) def test_timeseries_shape(self, reader, order): - timeseries = reader.timeseries(order=order) - a_index = order.index("a") - # f_index = order.index("f") - c_index = order.index("c") - assert timeseries.shape[a_index] == reader.n_atoms - # assert timeseries.shape[f_index] == len(reader) - assert timeseries.shape[c_index] == 3 - - @pytest.mark.skip( - reason="Timeseries currently requires n_frames to be known" - ) - @pytest.mark.parametrize("asel", ("index 1", "index 2", "index 1 to 3")) + pass + def test_timeseries_asel_shape(self, reader, asel): - atoms = mda.Universe(reader.filename).select_atoms(asel) - timeseries = reader.timeseries(atoms, order="fac") - assert timeseries.shape[0] == len(reader) - assert timeseries.shape[1] == len(atoms) - assert timeseries.shape[2] == 3 - - @pytest.mark.skip("Cannot slice stream") - @pytest.mark.parametrize("slice", ([0, 2, 1], [0, 10, 2], [0, 10, 3])) + pass + def test_timeseries_values(self, reader, slice): - ts_positions = [] - if isinstance(reader, mda.coordinates.memory.MemoryReader): - pytest.xfail( - "MemoryReader uses deprecated stop inclusive" - " indexing, see Issue #3893" - ) - if slice[1] > len(reader): - pytest.skip("too few frames in reader") - for i in range(slice[0], slice[1], slice[2]): - ts = reader[i] - ts_positions.append(ts.positions.copy()) - positions = np.asarray(ts_positions) - timeseries = reader.timeseries( - start=slice[0], - stop=slice[1], - step=slice[2], - order="fac", - ) - assert_allclose(timeseries, positions) + pass - @pytest.mark.skip(reason="Cannot rewind stream") def test_transformations_2iter(self, ref, transformed): - # Are the transformations applied and - # are the coordinates "overtransformed"? - v1 = np.float32((1, 1, 1)) - v2 = np.float32((0, 0, 0.33)) - idealcoords = [] - for i, ts in enumerate(transformed): - idealcoords.append(ref.iter_ts(i).positions + v1 + v2) - assert_array_almost_equal( - ts.positions, - idealcoords[i], - decimal=ref.prec, - ) + pass - for i, ts in enumerate(transformed): - assert_almost_equal( - ts.positions, - idealcoords[i], - decimal=ref.prec, - ) - - @pytest.mark.skip(reason="Cannot slice stream") def test_transformations_slice(self, ref, transformed): - # Are the transformations applied when iterating over a slice of the trajectory? - v1 = np.float32((1, 1, 1)) - v2 = np.float32((0, 0, 0.33)) - for i, ts in enumerate(transformed[2:3:1]): - idealcoords = ref.iter_ts(ts.frame).positions + v1 + v2 - assert_array_almost_equal( - ts.positions, idealcoords, decimal=ref.prec - ) + pass - @pytest.mark.skip(reason="Cannot slice stream") def test_transformations_switch_frame(self, ref, transformed): - # This test checks if the transformations are applied and if the coordinates - # "overtransformed" on different situations - # Are the transformations applied when we switch to a different frame? - v1 = np.float32((1, 1, 1)) - v2 = np.float32((0, 0, 0.33)) - first_ideal = ref.iter_ts(0).positions + v1 + v2 - if len(transformed) > 1: - assert_array_almost_equal( - transformed[0].positions, - first_ideal, - decimal=ref.prec, - ) - second_ideal = ref.iter_ts(1).positions + v1 + v2 - assert_array_almost_equal( - transformed[1].positions, - second_ideal, - decimal=ref.prec, - ) - - # What if we comeback to the previous frame? - assert_array_almost_equal( - transformed[0].positions, - first_ideal, - decimal=ref.prec, - ) - - # How about we switch the frame to itself? - assert_array_almost_equal( - transformed[0].positions, - first_ideal, - decimal=ref.prec, - ) - else: - assert_array_almost_equal( - transformed[0].positions, - first_ideal, - decimal=ref.prec, - ) + pass - @pytest.mark.skip(reason="Cannot rewind stream") def test_transformation_rewind(self, ref, transformed): - # this test checks if the transformations are applied after rewinding the - # trajectory - v1 = np.float32((1, 1, 1)) - v2 = np.float32((0, 0, 0.33)) - ideal_coords = ref.iter_ts(0).positions + v1 + v2 - transformed.rewind() - assert_array_almost_equal( - transformed[0].positions, - ideal_coords, - decimal=ref.prec, - ) + pass - @pytest.mark.skip(reason="Cannot make a copy of a stream") def test_copy(self, ref, transformed): - # this test checks if transformations are carried over a copy and if the - # coordinates of the copy are equal to the original's - v1 = np.float32((1, 1, 1)) - v2 = np.float32((0, 0, 0.33)) - new = transformed.copy() - assert_equal( - transformed.transformations, - new.transformations, - "transformations are not equal", - ) - for i, ts in enumerate(new): - ideal_coords = ref.iter_ts(i).positions + v1 + v2 - assert_array_almost_equal( - ts.positions, ideal_coords, decimal=ref.prec - ) + pass - @pytest.mark.skip(reason="Cannot pickle socket") def test_pickle_reader(self, reader): - """It probably wouldn't be a good idea to pickle a - reader that is connected to a server""" - reader_p = pickle.loads(pickle.dumps(reader)) - assert_equal(len(reader), len(reader_p)) - assert_equal( - reader.ts, - reader_p.ts, - "Timestep is changed after pickling", - ) + pass - @pytest.mark.skip(reason="Cannot pickle socket") def test_pickle_next_ts_reader(self, reader): - reader_p = pickle.loads(pickle.dumps(reader)) - assert_equal( - next(reader), - next(reader_p), - "Next timestep is changed after pickling", - ) + pass - @pytest.mark.skip(reason="Cannot pickle socket") def test_pickle_last_ts_reader(self, reader): - # move current ts to last frame. - reader[-1] - reader_p = pickle.loads(pickle.dumps(reader)) - assert_equal( - len(reader), - len(reader_p), - "Last timestep is changed after pickling", - ) - assert_equal( - reader.ts, - reader_p.ts, - "Last timestep is changed after pickling", - ) + pass - @pytest.mark.skip(reason="Cannot copy stream") def test_transformations_copy(self, ref, transformed): - # this test checks if transformations are carried over a copy and if the - # coordinates of the copy are equal to the original's - v1 = np.float32((1, 1, 1)) - v2 = np.float32((0, 0, 0.33)) - new = transformed.copy() - assert_equal( - transformed.transformations, - new.transformations, - "transformations are not equal", - ) - for i, ts in enumerate(new): - ideal_coords = ref.iter_ts(i).positions + v1 + v2 - assert_array_almost_equal( - ts.positions, ideal_coords, decimal=ref.prec - ) + pass - @pytest.mark.skip( - reason="Timeseries currently requires n_frames to be known" - ) def test_timeseries_empty_asel(self, reader): - with pytest.warns( - UserWarning, - match="Empty string to select atoms, empty group returned.", - ): - atoms = mda.Universe(reader.filename).select_atoms(None) - with pytest.raises(ValueError, match="Timeseries requires at least"): - reader.timeseries(asel=atoms) - - @pytest.mark.skip( - reason="Timeseries currently requires n_frames to be known" - ) + pass + def test_timeseries_empty_atomgroup(self, reader): - with pytest.warns( - UserWarning, - match="Empty string to select atoms, empty group returned.", - ): - atoms = mda.Universe(reader.filename).select_atoms(None) - with pytest.raises(ValueError, match="Timeseries requires at least"): - reader.timeseries(atomgroup=atoms) - - @pytest.mark.skip( - reason="Timeseries currently requires n_frames to be known" - ) + pass + def test_timeseries_asel_warns_deprecation(self, reader): - atoms = mda.Universe(reader.filename).select_atoms("index 1") - with pytest.warns(DeprecationWarning, match="asel argument to"): - timeseries = reader.timeseries(asel=atoms, order="fac") + pass - @pytest.mark.skip( - reason="Timeseries currently requires n_frames to be known" - ) def test_timeseries_atomgroup(self, reader): - atoms = mda.Universe(reader.filename).select_atoms("index 1") - timeseries = reader.timeseries(atomgroup=atoms, order="fac") + pass - @pytest.mark.skip( - reason="Timeseries currently requires n_frames to be known" - ) def test_timeseries_atomgroup_asel_mutex(self, reader): - atoms = mda.Universe(reader.filename).select_atoms("index 1") - with pytest.raises(ValueError, match="Cannot provide both"): - timeseries = reader.timeseries( - atomgroup=atoms, asel=atoms, order="fac" - ) + pass - @pytest.mark.skip("Cannot slice stream") def test_last_frame(self, ref, reader): - ts = reader[-1] - assert_timestep_almost_equal(ts, ref.last_frame, decimal=ref.prec) + pass - @pytest.mark.skip("Cannot slice stream") def test_go_over_last_frame(self, ref, reader): - with pytest.raises(IndexError): - reader[ref.n_frames + 1] + pass - @pytest.mark.skip("Cannot slice stream") def test_frame_jump(self, ref, reader): - ts = reader[ref.jump_to_frame.frame] - assert_timestep_almost_equal(ts, ref.jump_to_frame, decimal=ref.prec) + pass - @pytest.mark.skip("Cannot slice stream") def test_frame_jump_issue1942(self, ref, reader): - """Test for issue 1942 (especially XDR on macOS)""" - reader.rewind() - try: - for ii in range(ref.n_frames + 2): - reader[0] - except StopIteration: - pytest.fail("Frame-seeking wrongly iterated (#1942)") + pass def test_next_gives_second_frame(self, ref, reader): # don't recreate reader here as in inherited base test ts = reader.next() assert_timestep_almost_equal(ts, ref.second_frame, decimal=ref.prec) - @pytest.mark.skip( - reason="Stream isn't rewound after iteration- base reference is the same but it is the last frame" - ) def test_frame_collect_all_same(self, reader): - # check that the timestep resets so that the base reference is the same - # for all timesteps in a collection with the exception of memoryreader - # and DCDReader - if isinstance(reader, mda.coordinates.memory.MemoryReader): - pytest.xfail("memoryreader allows independent coordinates") - if isinstance(reader, mda.coordinates.DCD.DCDReader): - pytest.xfail( - "DCDReader allows independent coordinates." - "This behaviour is deprecated and will be changed" - "in 3.0" - ) - collected_ts = [] - for i, ts in enumerate(reader): - collected_ts.append(ts.positions) - for array in collected_ts: - assert_allclose(array, collected_ts[0]) + pass @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not isntalled") From bc88b7b921f5211a12688567f32a7e52108a1e7a Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Mon, 5 May 2025 11:11:20 -0400 Subject: [PATCH 004/114] bump CI From cf15cf9cf6701d58b171b06d47e563c8f6f5c105 Mon Sep 17 00:00:00 2001 From: "Jennifer A. Clark" Date: Sat, 24 May 2025 19:50:53 -0400 Subject: [PATCH 005/114] Add minimum docs --- package/MDAnalysis/coordinates/IMD.py | 32 +++++++++++++++++-- package/MDAnalysis/coordinates/__init__.py | 5 +++ .../documentation_pages/coordinates/IMD.rst | 1 + .../coordinates_modules.rst | 1 + .../source/documentation_pages/references.rst | 7 ++++ 5 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 package/doc/sphinx/source/documentation_pages/coordinates/IMD.rst diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 9675e5bcd0d..943958063b0 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -1,6 +1,34 @@ """ -MDAnalysis IMDReader -^^^^^^^^^^^^^^^^^^^^ +IMDReader --- :mod:`MDAnalysis.coordinates.IMD` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Read and analyze simulation data interactively using `IMDClient`_. + +.. _IMDClient: https://github.com/Becksteinlab/imdclient + +Units +----- +The units in IMDv3 are fixed. + +.. list-table:: + :widths: 10 10 + :header-rows: 1 + + * - Measurement + - Unit + * - Length + - angstrom + * - Velocity + - angstrom/picosecond + * - Force + - kilojoules/(mol*angstrom) + * - Time + - picosecond + * - Energy + - kilojoules/mol + +Classes +------- .. autoclass:: IMDReader :members: diff --git a/package/MDAnalysis/coordinates/__init__.py b/package/MDAnalysis/coordinates/__init__.py index b9a59af2483..8a791598ace 100644 --- a/package/MDAnalysis/coordinates/__init__.py +++ b/package/MDAnalysis/coordinates/__init__.py @@ -274,6 +274,11 @@ class can choose an appropriate reader automatically. | library | | | file formats`_ and | | | | | :mod:`MDAnalysis.coordinates.chemfiles` | +---------------+-----------+-------+------------------------------------------------------+ + | IMD | IP address| r/w | Receive simulation trajectory data using interactive | + | | and port | | molecular dynamics version 3 (IMDv3) by configuring | + | | number | | a socket address to a NAMD, GROMACS, or LAMMPS | + | | | | simulation. | + +---------------+-----------+-------+------------------------------------------------------+ .. [#a] This format can also be used to provide basic *topology* information (i.e. the list of atoms); it is possible to create a diff --git a/package/doc/sphinx/source/documentation_pages/coordinates/IMD.rst b/package/doc/sphinx/source/documentation_pages/coordinates/IMD.rst new file mode 100644 index 00000000000..d4d8013d61d --- /dev/null +++ b/package/doc/sphinx/source/documentation_pages/coordinates/IMD.rst @@ -0,0 +1 @@ +.. automodule:: MDAnalysis.coordinates.IMD \ No newline at end of file diff --git a/package/doc/sphinx/source/documentation_pages/coordinates_modules.rst b/package/doc/sphinx/source/documentation_pages/coordinates_modules.rst index af726d0f857..767a6f591aa 100644 --- a/package/doc/sphinx/source/documentation_pages/coordinates_modules.rst +++ b/package/doc/sphinx/source/documentation_pages/coordinates_modules.rst @@ -27,6 +27,7 @@ provide the format in the keyword argument *format* to coordinates/GSD coordinates/GRO coordinates/H5MD + coordinates/IMD coordinates/INPCRD coordinates/LAMMPS coordinates/MMTF diff --git a/package/doc/sphinx/source/documentation_pages/references.rst b/package/doc/sphinx/source/documentation_pages/references.rst index 7de33322fd3..55e0b408a49 100644 --- a/package/doc/sphinx/source/documentation_pages/references.rst +++ b/package/doc/sphinx/source/documentation_pages/references.rst @@ -229,6 +229,13 @@ If you use H5MD files using pp. 18 – 26, 2021. doi:`10.25080/majora-1b6fd038-005. `_ +If you use IMD capability with :mod:`MDAnalysis.coordinates.IMD.py`, please cite +[IMDv3paper]_. + +.. [IMDv3paper] Authors (YEAR). + IMDv3 Manuscript Title. + *Journal*, 185. doi:`insert-doi-here + `_ .. _citations-using-duecredit: From b5ff03d28bdf6561b28122e877d6f59b69a92da4 Mon Sep 17 00:00:00 2001 From: hmacdope Date: Sat, 7 Jun 2025 15:15:49 +1000 Subject: [PATCH 006/114] try add mock for import versions --- package/MDAnalysis/coordinates/IMD.py | 18 +++++++++++++++++ .../MDAnalysisTests/coordinates/test_imd.py | 20 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 943958063b0..f07acc4431c 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -38,16 +38,23 @@ import numpy as np import logging +import warnings from MDAnalysis.coordinates import core from MDAnalysis.lib.util import store_init_arguments from MDAnalysis.coordinates.base import StreamReaderBase + +from packaging.version import Version + +MIN_IMDCLIENT_VERSION = Version("0.1.4") + try: import imdclient from imdclient.IMDClient import IMDClient except ImportError: HAS_IMDCLIENT = False + imdclient_version = Version("0.0.0") # Allow building documentation without imdclient import types @@ -60,6 +67,17 @@ class MockIMDClient: else: HAS_IMDCLIENT = True + # Check for compatibility: currently needs to be >=0.1.4 + imdclient_version = Version(imdclient.__version__) + if imdclient_version < MIN_IMDCLIENT_VERSION: + warnings.warn( + f"imdclient version {imdclient_version} is too old; " + f"need at least {imdclient_version}, Your installed version of " + "distopia will NOT be used.", + category=RuntimeWarning, + ) + HAS_IMDCLIENT = False + logger = logging.getLogger("MDAnalysis.coordinates.IMDReader") diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 753453e023c..159b850ff33 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -12,6 +12,8 @@ assert_equal, assert_allclose, ) +import sys +from types import ModuleType from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT if HAS_IMDCLIENT: @@ -36,6 +38,24 @@ assert_timestep_almost_equal, ) + +def test_HAS_IMDCLIENT(): + # mock a version of imdclient that is too old + module_name = "imdclient" + + sys.modules.pop(module_name, None) + sys.modules.pop("MDAnalysis.coordinates.IMD", None) + + mocked_module = ModuleType(module_name) + # too old version + mocked_module.__version__ = "0.1.0" + sys.modules[module_name] = mocked_module + + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + assert HAS_IMDCLIENT + + + @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class IMDReference(BaseReference): def __init__(self): From 62631510ce060ef48c1eda789964f77cec5c2d7c Mon Sep 17 00:00:00 2001 From: hmacdope Date: Sat, 7 Jun 2025 15:31:23 +1000 Subject: [PATCH 007/114] fix for wrong assert --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 159b850ff33..a11f6381f7f 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -51,11 +51,22 @@ def test_HAS_IMDCLIENT(): mocked_module.__version__ = "0.1.0" sys.modules[module_name] = mocked_module + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + assert not HAS_IMDCLIENT + + sys.modules.pop(module_name, None) + sys.modules.pop("MDAnalysis.coordinates.IMD", None) + + # new enough version + mocked_module.__version__ = "0.1.4" + sys.modules[module_name] = mocked_module + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT assert HAS_IMDCLIENT + @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class IMDReference(BaseReference): def __init__(self): From efbb903eeb4a622a3802ee050f173b3589574ef7 Mon Sep 17 00:00:00 2001 From: hmacdope Date: Sat, 7 Jun 2025 16:20:35 +1000 Subject: [PATCH 008/114] fix fixture issues --- package/MDAnalysis/coordinates/IMD.py | 5 +++-- testsuite/MDAnalysisTests/coordinates/test_imd.py | 15 ++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index f07acc4431c..6c09f4b3e2d 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -63,17 +63,18 @@ class MockIMDClient: pass imdclient = types.ModuleType("imdclient") imdclient.IMDClient = MockIMDClient + imdclient.__version__ = "0.0.0" else: HAS_IMDCLIENT = True + imdclient_version = Version(imdclient.__version__) # Check for compatibility: currently needs to be >=0.1.4 - imdclient_version = Version(imdclient.__version__) if imdclient_version < MIN_IMDCLIENT_VERSION: warnings.warn( f"imdclient version {imdclient_version} is too old; " f"need at least {imdclient_version}, Your installed version of " - "distopia will NOT be used.", + "imdclient will NOT be used.", category=RuntimeWarning, ) HAS_IMDCLIENT = False diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index a11f6381f7f..c87e80456e3 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -39,7 +39,7 @@ ) -def test_HAS_IMDCLIENT(): +def test_HAS_IMDCLIENT_too_old(): # mock a version of imdclient that is too old module_name = "imdclient" @@ -54,9 +54,14 @@ def test_HAS_IMDCLIENT(): from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT assert not HAS_IMDCLIENT + + +def test_HAS_IMDCLIENT_new_enough(): + module_name = "imdclient" sys.modules.pop(module_name, None) sys.modules.pop("MDAnalysis.coordinates.IMD", None) + mocked_module = ModuleType(module_name) # new enough version mocked_module.__version__ = "0.1.4" sys.modules[module_name] = mocked_module @@ -206,16 +211,16 @@ def test_reload_auxiliaries_from_description(self, ref, reader): def test_stop_iter(self, reader): pass - def test_iter_rewinds(self, reader, accessor): + def test_iter_rewinds(self, reader): pass - def test_timeseries_shape(self, reader, order): + def test_timeseries_shape(self, reader,): pass - def test_timeseries_asel_shape(self, reader, asel): + def test_timeseries_asel_shape(self, reader): pass - def test_timeseries_values(self, reader, slice): + def test_timeseries_values(self, reader): pass def test_transformations_2iter(self, ref, transformed): From 2ff3935c5ce8e9758ff7cb2ef2a1d055e13fca0e Mon Sep 17 00:00:00 2001 From: hmacdope Date: Sat, 7 Jun 2025 16:33:34 +1000 Subject: [PATCH 009/114] add short class description --- package/MDAnalysis/coordinates/base.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 77a805f4158..ecc299063e6 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1846,6 +1846,14 @@ def convert(self, obj): raise NotImplementedError class StreamReaderBase(ReaderBase): + """Base class for readers that read a continuous stream of data. + + This class is used for readers that read a continuous stream of data, + such as a live feed from a simulation. This places some constraints on the + reader, such as not being able to rewind or iterate more than once. + + .. versionadded:: 2.9.0 + """ def __init__(self, filename, convert_units=True, **kwargs): super(StreamReaderBase, self).__init__( From 073430b16395f9fba44f755aba7c092eb93c9c87 Mon Sep 17 00:00:00 2001 From: hmacdope Date: Sat, 7 Jun 2025 16:48:15 +1000 Subject: [PATCH 010/114] add stricter matching to tests --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index c87e80456e3..56d65804ac7 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -324,35 +324,36 @@ def test_iterate_step(self, reader, universe): def test_iterate_twice_sliced_raises_error(self, reader): for ts in reader[::2]: pass - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError, match="Cannot reopen stream"): for ts in reader[::2]: pass def test_iterate_twice_all_raises_error(self, reader): for ts in reader: pass - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError, match="Cannot reopen stream"): for ts in reader: pass def test_iterate_twice_fi_all_raises_error(self, reader): for ts in reader[:]: pass - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError, match="Cannot reopen stream"): for ts in reader[:]: pass def test_index_stream_raises_error(self, reader): - with pytest.raises(TypeError): + with pytest.raises(TypeError, match="Streamed trajectories must be"): reader[0] def test_iterate_backwards_raises_error(self, reader): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Cannot go backwards"): for ts in reader[::-1]: pass + def test_iterate_start_stop_raises_error(self, reader): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Cannot expect a start index"): for ts in reader[1:3]: pass From 45ad9211390031d9fcb98d7d3946402571a22ba8 Mon Sep 17 00:00:00 2001 From: hmacdope Date: Sat, 7 Jun 2025 16:56:22 +1000 Subject: [PATCH 011/114] forbid use of .timeseries() method for streamed trajectories --- package/MDAnalysis/coordinates/base.py | 7 ++++++- testsuite/MDAnalysisTests/coordinates/test_imd.py | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index ecc299063e6..6a7ea245527 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1919,7 +1919,7 @@ def rewind(self): # Incompatible methods def copy(self): - raise NotImplementedError( + raise RuntimeError( "{} does not support copying".format(self.__class__.__name__) ) @@ -1931,6 +1931,11 @@ def _reopen(self): self._frame = -1 self._reopen_called = True + def timeseries(self, **kwargs): + raise RuntimeError( + "{}: cannot access timeseries for streamed trajectories".format(self.__class__.__name__) + ) + def __getitem__(self, frame): """Return the Timestep corresponding to *frame*. diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 56d65804ac7..64382a70cbb 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -366,6 +366,13 @@ def test_subslice_fi_all_after_iteration_raises_error(self, reader): for ts in sub_sliced_reader: pass + + def test_timeseries_raises(self, reader): + with pytest.raises( + RuntimeError, match="cannot access timeseries for streamed trajectories" + ): + reader.timeseries() + @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") def test_n_atoms_mismatch(): universe = mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) From eb825c6c5bf0735ff81fc85a1eca17ad8ca21c07 Mon Sep 17 00:00:00 2001 From: "Jennifer A. Clark" Date: Thu, 19 Jun 2025 18:14:19 -0400 Subject: [PATCH 012/114] Update test_HAS_IMDCLIENT_new_enough to pass --- .../MDAnalysisTests/coordinates/test_imd.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 64382a70cbb..0177688161b 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -1,19 +1,21 @@ """Test for MDAnalysis trajectory reader expectations """ -import numpy as np -import logging + +import sys import pytest -from MDAnalysis.transformations import translate import pickle -import MDAnalysis as mda +from types import ModuleType + +import numpy as np from numpy.testing import ( assert_almost_equal, assert_array_almost_equal, assert_equal, assert_allclose, ) -import sys -from types import ModuleType + +from MDAnalysis.transformations import translate +import MDAnalysis as mda from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT if HAS_IMDCLIENT: @@ -62,16 +64,24 @@ def test_HAS_IMDCLIENT_new_enough(): sys.modules.pop("MDAnalysis.coordinates.IMD", None) mocked_module = ModuleType(module_name) + IMDClient_module = ModuleType(f"{module_name}.IMDClient") + + class MockIMDClient: + pass + + IMDClient_module.IMDClient = MockIMDClient + mocked_module.IMDClient = IMDClient_module # new enough version mocked_module.__version__ = "0.1.4" + sys.modules[module_name] = mocked_module + sys.modules[f"{module_name}.IMDClient"] = IMDClient_module from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT assert HAS_IMDCLIENT - @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class IMDReference(BaseReference): def __init__(self): From a8b41575c5cfbf1831171f42f6df173031af9927 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Mon, 23 Jun 2025 11:39:02 -0700 Subject: [PATCH 013/114] add imd docs --- package/MDAnalysis/coordinates/IMD.py | 35 ++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 6c09f4b3e2d..49d7a0c7972 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -2,9 +2,35 @@ IMDReader --- :mod:`MDAnalysis.coordinates.IMD` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Read and analyze simulation data interactively using `IMDClient`_. +:class:`MDAnalysis.coordinates.IMD.IMDReader` is a class that implements the Interactive Molecular Dynamics (IMD) protocol for reading simulation +data using the IMDClient (see `imdclient `_). +The protocol allows two-way communicating molecular simulation data through a socket. +Via IMD, a simulation engine sends data to a receiver (in this case, the IMDClient) and the receiver can send forces and specific control +requests (such as pausing, resuming, or terminating the simulation) back to the simulation engine. It currently supports +simulation running with GROMACS, LAMMPS, or NAMD. + +For example, when running a simulation with GROMACS that supports streaming, use the following commands: + +.. code-block:: bash + + gmx grompp -f run-NPT_imd-v3.mdp -c conf.gro -p topol.top -o topol.tpr + gmx mdrun -v -nt 4 -imdwait -imdport 8889 + +The :class:`MDAnalysis.coordinates.IMD.IMDReader` can then connect to the running simulation and stream data in real time: + +.. code-block:: python + + import MDAnalysis as mda + u = mda.Universe("topol.tpr", "imd://localhost:8889", buffer_size=10*1024*1024) + + print(" time [ position ] [ velocity ] [ force ] [ box ]") + sel = u.select_atoms("all") # Select all atoms; adjust selection as needed + for ts in u.trajectory: + print(f'{ts.time:8.3f} {sel[0].position} {sel[0].velocity} {sel[0].force} {u.dimensions[0:3]}') + +Details about the IMD protocol and usage examples can be found in the +`imdclient `_ repository. -.. _IMDClient: https://github.com/Becksteinlab/imdclient Units ----- @@ -84,7 +110,8 @@ class MockIMDClient: class IMDReader(StreamReaderBase): """ - Reader for IMD protocol packets. + Reader that supports the Interactive Molecular Dynamics (IMD) protocol for reading simulation + data using the IMDClient. Parameters ---------- @@ -186,7 +213,7 @@ def _load_imdframe_into_ts(self, imdf): def _format_hint(thing): try: parse_host_port(thing) - except: + except ValueError: return False return HAS_IMDCLIENT and True From 0dfb1944f330ac3c71a68b40bf19abce68ff1850 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Mon, 23 Jun 2025 11:48:17 -0700 Subject: [PATCH 014/114] add doc for streamreader --- package/MDAnalysis/coordinates/base.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 6a7ea245527..df4c9e03240 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -50,6 +50,7 @@ .. autoclass:: FrameIteratorIndices +.. autoclass:: StreamFrameIteratorSliced .. _ReadersBase: @@ -87,8 +88,10 @@ .. autoclass:: ProtoReader :members: +.. autoclass:: StreamReaderBase + :members: - + .. _WritersBase: Writers @@ -1852,7 +1855,7 @@ class StreamReaderBase(ReaderBase): such as a live feed from a simulation. This places some constraints on the reader, such as not being able to rewind or iterate more than once. - .. versionadded:: 2.9.0 + .. versionadded:: 2.10.0 """ def __init__(self, filename, convert_units=True, **kwargs): @@ -2014,6 +2017,8 @@ def __repr__(self): class StreamFrameIteratorSliced(FrameIteratorBase): + """Iterator for sliced frames in a streamed trajectory. + """ def __init__(self, trajectory, step): super().__init__(trajectory) From 83d9443bb1f455ce26c4f7ee89034636c4a1b189 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Mon, 23 Jun 2025 11:51:31 -0700 Subject: [PATCH 015/114] black imd --- package/MDAnalysis/coordinates/IMD.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 49d7a0c7972..e676d9a240e 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -87,6 +87,7 @@ class MockIMDClient: pass + imdclient = types.ModuleType("imdclient") imdclient.IMDClient = MockIMDClient imdclient.__version__ = "0.0.0" @@ -225,10 +226,13 @@ def close(self): # NOTE: removeme after testing logger.debug("IMDReader shut down gracefully.") + # NOTE: think of other edge cases as well- should be robust def parse_host_port(filename): if not filename.startswith("imd://"): - raise ValueError("IMDReader: URL must be in the format 'imd://host:port'") + raise ValueError( + "IMDReader: URL must be in the format 'imd://host:port'" + ) # Check if the format is correct parts = filename.split("imd://")[1].split(":") if len(parts) == 2: @@ -239,4 +243,6 @@ def parse_host_port(filename): except ValueError as e: raise ValueError("IMDReader: Port must be an integer") from e else: - raise ValueError("IMDReader: URL must be in the format 'imd://host:port'") + raise ValueError( + "IMDReader: URL must be in the format 'imd://host:port'" + ) From a169cb623d0b792b97fdb68e38380766289f5d58 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Mon, 23 Jun 2025 12:30:59 -0700 Subject: [PATCH 016/114] add reason for skipped test --- .../MDAnalysisTests/coordinates/test_imd.py | 76 +++++++++---------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 0177688161b..991f6dc2524 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -170,8 +170,8 @@ def transformed(self, ref): ) return transformed - def test_n_frames(self, reader, ref): - pass + def test_n_frames(self, ref, reader): + pytest.skip('`n_frames` is unknown for IMDReader') def test_first_frame(self, ref, reader): # don't rewind here as in inherited base test @@ -180,16 +180,16 @@ def test_first_frame(self, ref, reader): ) def test_get_writer_1(self, ref, reader, tmpdir): - pass + pytest.skip('No Writer for IMDReader') def test_get_writer_2(self, ref, reader, tmpdir): - pass + pytest.skip('No Writer for IMDReader') - def test_total_time(self, reader, ref): - pass + def test_total_time(self, ref, reader): + pytest.skip('`total_time` is unknown for IMDReader') def test_changing_dimensions(self, ref, reader): - pass + pytest.skip('IMDReader cannot be rewound') def test_iter(self, ref, reader): for i, ts in enumerate(reader): @@ -207,7 +207,6 @@ def test_first_dimensions(self, ref, reader): atol=1.5 * 10**(-ref.prec) ) - def test_volume(self, ref, reader): # don't rewind here as in inherited base test vol = reader.ts.volume @@ -216,76 +215,73 @@ def test_volume(self, ref, reader): assert_allclose(vol, ref.volume, rtol=0, atol=1.5e0) def test_reload_auxiliaries_from_description(self, ref, reader): - pass + pytest.skip('Cannot create two IMDReaders on the same stream') def test_stop_iter(self, reader): - pass + pytest.skip('IMDReader cannot be rewound') - def test_iter_rewinds(self, reader): - pass + def test_iter_rewinds(self, reader, accessor): + pytest.skip('IMDReader cannot be rewound') - def test_timeseries_shape(self, reader,): - pass + def test_timeseries_shape(self, reader, order): + pytest.skip('IMDReader does not support timeseries') - def test_timeseries_asel_shape(self, reader): - pass + def test_timeseries_asel_shape(self, reader, asel): + pytest.skip('IMDReader does not support timeseries') - def test_timeseries_values(self, reader): - pass + def test_timeseries_values(self, reader, slice): + pytest.skip('IMDReader does not support timeseries') def test_transformations_2iter(self, ref, transformed): - pass + pytest.skip('IMDReader cannot be reopened') def test_transformations_slice(self, ref, transformed): - pass + pytest.skip('IMDReader cannot be reopened') def test_transformations_switch_frame(self, ref, transformed): - pass + pytest.skip('IMDReader cannot be reopened') def test_transformation_rewind(self, ref, transformed): - pass - - def test_copy(self, ref, transformed): - pass + pytest.skip('IMDReader cannot be reopened') def test_pickle_reader(self, reader): - pass + pytest.skip('IMDReader cannot be pickled') def test_pickle_next_ts_reader(self, reader): - pass + pytest.skip('IMDReader cannot be pickled') def test_pickle_last_ts_reader(self, reader): - pass + pytest.skip('IMDReader cannot be pickled') def test_transformations_copy(self, ref, transformed): - pass + pytest.skip('IMDReader cannot be copied') def test_timeseries_empty_asel(self, reader): - pass + pytest.skip('IMDReader does not support timeseries') def test_timeseries_empty_atomgroup(self, reader): - pass + pytest.skip('IMDReader does not support timeseries') def test_timeseries_asel_warns_deprecation(self, reader): - pass + pytest.skip('IMDReader does not support timeseries') def test_timeseries_atomgroup(self, reader): - pass + pytest.skip('IMDReader does not support timeseries') def test_timeseries_atomgroup_asel_mutex(self, reader): - pass + pytest.skip('IMDReader does not support timeseries') def test_last_frame(self, ref, reader): - pass + pytest.skip('IMDReader cannot be rewound') def test_go_over_last_frame(self, ref, reader): - pass + pytest.skip('IMDReader must be an indexed using a slice') def test_frame_jump(self, ref, reader): - pass + pytest.skip('IMDReader must be an indexed using a slice') def test_frame_jump_issue1942(self, ref, reader): - pass + pytest.skip('IMDReader must be an indexed using a slice') def test_next_gives_second_frame(self, ref, reader): # don't recreate reader here as in inherited base test @@ -293,10 +289,10 @@ def test_next_gives_second_frame(self, ref, reader): assert_timestep_almost_equal(ts, ref.second_frame, decimal=ref.prec) def test_frame_collect_all_same(self, reader): - pass + pytest.skip('IMDReader has independent coordinates') -@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not isntalled") +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class TestStreamIteration: @pytest.fixture From d48fec100b53de66b7da36375e26c422785de155 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Mon, 23 Jun 2025 12:37:19 -0700 Subject: [PATCH 017/114] use staticmethod for readers --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 991f6dc2524..f1a19cb001c 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -130,8 +130,9 @@ def ref(self): """Not a static method like in base class- need new server for each test""" return IMDReference() + @staticmethod @pytest.fixture() - def reader(self, ref): + def reader(ref): # This will start the test IMD Server, waiting for a connection # to then send handshake & first frame ref.server.handshake_sequence("localhost", ref.port) @@ -156,8 +157,9 @@ def reader(self, ref): ) return reader + @staticmethod @pytest.fixture() - def transformed(self, ref): + def transformed(ref): # This will start the test IMD Server, waiting for a connection # to then send handshake & first frame ref.server.handshake_sequence("localhost", ref.port) @@ -220,16 +222,16 @@ def test_reload_auxiliaries_from_description(self, ref, reader): def test_stop_iter(self, reader): pytest.skip('IMDReader cannot be rewound') - def test_iter_rewinds(self, reader, accessor): + def test_iter_rewinds(self, reader): pytest.skip('IMDReader cannot be rewound') - def test_timeseries_shape(self, reader, order): + def test_timeseries_shape(self, reader): pytest.skip('IMDReader does not support timeseries') - def test_timeseries_asel_shape(self, reader, asel): + def test_timeseries_asel_shape(self, reader): pytest.skip('IMDReader does not support timeseries') - def test_timeseries_values(self, reader, slice): + def test_timeseries_values(self, reader): pytest.skip('IMDReader does not support timeseries') def test_transformations_2iter(self, ref, transformed): From c23fa3eb7d0d1a623140ddc8067d369604bedf35 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Mon, 23 Jun 2025 12:38:58 -0700 Subject: [PATCH 018/114] black imd test --- .../MDAnalysisTests/coordinates/test_imd.py | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index f1a19cb001c..db9d8b8afcd 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -54,8 +54,8 @@ def test_HAS_IMDCLIENT_too_old(): sys.modules[module_name] = mocked_module from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - assert not HAS_IMDCLIENT + assert not HAS_IMDCLIENT def test_HAS_IMDCLIENT_new_enough(): @@ -78,8 +78,8 @@ class MockIMDClient: sys.modules[f"{module_name}.IMDClient"] = IMDClient_module from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - assert HAS_IMDCLIENT + assert HAS_IMDCLIENT @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") @@ -125,7 +125,7 @@ def iter_ts(self, i): @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class TestIMDReaderBaseAPI(MultiframeReaderTest): - @pytest.fixture(scope='function') + @pytest.fixture(scope="function") def ref(self): """Not a static method like in base class- need new server for each test""" return IMDReference() @@ -173,7 +173,7 @@ def transformed(ref): return transformed def test_n_frames(self, ref, reader): - pytest.skip('`n_frames` is unknown for IMDReader') + pytest.skip("`n_frames` is unknown for IMDReader") def test_first_frame(self, ref, reader): # don't rewind here as in inherited base test @@ -182,16 +182,16 @@ def test_first_frame(self, ref, reader): ) def test_get_writer_1(self, ref, reader, tmpdir): - pytest.skip('No Writer for IMDReader') + pytest.skip("No Writer for IMDReader") def test_get_writer_2(self, ref, reader, tmpdir): - pytest.skip('No Writer for IMDReader') + pytest.skip("No Writer for IMDReader") def test_total_time(self, ref, reader): - pytest.skip('`total_time` is unknown for IMDReader') + pytest.skip("`total_time` is unknown for IMDReader") def test_changing_dimensions(self, ref, reader): - pytest.skip('IMDReader cannot be rewound') + pytest.skip("IMDReader cannot be rewound") def test_iter(self, ref, reader): for i, ts in enumerate(reader): @@ -206,7 +206,7 @@ def test_first_dimensions(self, ref, reader): reader.ts.dimensions, ref.dimensions, rtol=0, - atol=1.5 * 10**(-ref.prec) + atol=1.5 * 10 ** (-ref.prec), ) def test_volume(self, ref, reader): @@ -217,73 +217,73 @@ def test_volume(self, ref, reader): assert_allclose(vol, ref.volume, rtol=0, atol=1.5e0) def test_reload_auxiliaries_from_description(self, ref, reader): - pytest.skip('Cannot create two IMDReaders on the same stream') + pytest.skip("Cannot create two IMDReaders on the same stream") def test_stop_iter(self, reader): - pytest.skip('IMDReader cannot be rewound') + pytest.skip("IMDReader cannot be rewound") def test_iter_rewinds(self, reader): - pytest.skip('IMDReader cannot be rewound') + pytest.skip("IMDReader cannot be rewound") def test_timeseries_shape(self, reader): - pytest.skip('IMDReader does not support timeseries') + pytest.skip("IMDReader does not support timeseries") def test_timeseries_asel_shape(self, reader): - pytest.skip('IMDReader does not support timeseries') + pytest.skip("IMDReader does not support timeseries") def test_timeseries_values(self, reader): - pytest.skip('IMDReader does not support timeseries') + pytest.skip("IMDReader does not support timeseries") def test_transformations_2iter(self, ref, transformed): - pytest.skip('IMDReader cannot be reopened') + pytest.skip("IMDReader cannot be reopened") def test_transformations_slice(self, ref, transformed): - pytest.skip('IMDReader cannot be reopened') + pytest.skip("IMDReader cannot be reopened") def test_transformations_switch_frame(self, ref, transformed): - pytest.skip('IMDReader cannot be reopened') + pytest.skip("IMDReader cannot be reopened") def test_transformation_rewind(self, ref, transformed): - pytest.skip('IMDReader cannot be reopened') + pytest.skip("IMDReader cannot be reopened") def test_pickle_reader(self, reader): - pytest.skip('IMDReader cannot be pickled') + pytest.skip("IMDReader cannot be pickled") def test_pickle_next_ts_reader(self, reader): - pytest.skip('IMDReader cannot be pickled') + pytest.skip("IMDReader cannot be pickled") def test_pickle_last_ts_reader(self, reader): - pytest.skip('IMDReader cannot be pickled') + pytest.skip("IMDReader cannot be pickled") def test_transformations_copy(self, ref, transformed): - pytest.skip('IMDReader cannot be copied') + pytest.skip("IMDReader cannot be copied") def test_timeseries_empty_asel(self, reader): - pytest.skip('IMDReader does not support timeseries') + pytest.skip("IMDReader does not support timeseries") def test_timeseries_empty_atomgroup(self, reader): - pytest.skip('IMDReader does not support timeseries') + pytest.skip("IMDReader does not support timeseries") def test_timeseries_asel_warns_deprecation(self, reader): - pytest.skip('IMDReader does not support timeseries') + pytest.skip("IMDReader does not support timeseries") def test_timeseries_atomgroup(self, reader): - pytest.skip('IMDReader does not support timeseries') + pytest.skip("IMDReader does not support timeseries") def test_timeseries_atomgroup_asel_mutex(self, reader): - pytest.skip('IMDReader does not support timeseries') + pytest.skip("IMDReader does not support timeseries") def test_last_frame(self, ref, reader): - pytest.skip('IMDReader cannot be rewound') + pytest.skip("IMDReader cannot be rewound") def test_go_over_last_frame(self, ref, reader): - pytest.skip('IMDReader must be an indexed using a slice') + pytest.skip("IMDReader must be an indexed using a slice") def test_frame_jump(self, ref, reader): - pytest.skip('IMDReader must be an indexed using a slice') + pytest.skip("IMDReader must be an indexed using a slice") def test_frame_jump_issue1942(self, ref, reader): - pytest.skip('IMDReader must be an indexed using a slice') + pytest.skip("IMDReader must be an indexed using a slice") def test_next_gives_second_frame(self, ref, reader): # don't recreate reader here as in inherited base test @@ -291,7 +291,7 @@ def test_next_gives_second_frame(self, ref, reader): assert_timestep_almost_equal(ts, ref.second_frame, decimal=ref.prec) def test_frame_collect_all_same(self, reader): - pytest.skip('IMDReader has independent coordinates') + pytest.skip("IMDReader has independent coordinates") @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") @@ -359,7 +359,6 @@ def test_iterate_backwards_raises_error(self, reader): for ts in reader[::-1]: pass - def test_iterate_start_stop_raises_error(self, reader): with pytest.raises(ValueError, match="Cannot expect a start index"): for ts in reader[1:3]: @@ -374,13 +373,14 @@ def test_subslice_fi_all_after_iteration_raises_error(self, reader): for ts in sub_sliced_reader: pass - def test_timeseries_raises(self, reader): with pytest.raises( - RuntimeError, match="cannot access timeseries for streamed trajectories" + RuntimeError, + match="cannot access timeseries for streamed trajectories", ): reader.timeseries() + @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") def test_n_atoms_mismatch(): universe = mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) @@ -395,4 +395,4 @@ def test_n_atoms_mismatch(): IMDReader( f"imd://localhost:{port}", n_atoms=universe.trajectory.n_atoms + 1, - ) \ No newline at end of file + ) From 809d592641a87688776d27e0ed8cd0719216ce56 Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Sat, 28 Jun 2025 19:17:28 +1000 Subject: [PATCH 019/114] try 0.2.0b0 imdclient --- .github/actions/setup-deps/action.yaml | 8 +++++--- azure-pipelines.yml | 3 ++- maintainer/conda/environment.yml | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-deps/action.yaml b/.github/actions/setup-deps/action.yaml index a2f85662d3d..e7e219ba695 100644 --- a/.github/actions/setup-deps/action.yaml +++ b/.github/actions/setup-deps/action.yaml @@ -82,8 +82,8 @@ inputs: default: 'seaborn>=0.7.0' tidynamics: default: 'tidynamics>=1.0.0' - imdclient: - default: 'imdclient' + # imdclient: + # default: 'imdclient' # pip-installed min dependencies coverage: default: 'coverage' @@ -108,6 +108,8 @@ inputs: default: 'pyedr>=0.7.0' waterdynamics: default: 'waterdynamics' + imdclient: + default: 'imdclient==0.2.0b0' runs: using: "composite" @@ -139,7 +141,6 @@ runs: ${{ inputs.distopia }} ${{ inputs.gsd }} ${{ inputs.h5py }} - ${{ inputs.imdclient }} ${{ inputs.hole2 }} ${{ inputs.joblib }} ${{ inputs.netcdf4 }} @@ -181,6 +182,7 @@ runs: ${{ inputs.duecredit }} ${{ inputs.parmed }} ${{ inputs.pyedr }} + ${{ inputs.imdclient}} run: | # setup full variable if [ ${{ inputs.full-deps }} = "true" ]; then diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 38c70fe8cc7..30a1e452bab 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -77,7 +77,6 @@ jobs: displayName: 'Install tools' - script: >- python -m pip install --only-binary=scipy,h5py - imdclient cython hypothesis h5py>=2.10 @@ -114,6 +113,8 @@ jobs: pytng>=0.2.3 rdkit>=2020.03.1 tidynamics>=1.0.0 + imdclient==0.2.0b0 + # remove from azure to avoid test hanging #4707 # "gsd>3.0.0" displayName: 'Install additional dependencies for 64-bit tests' diff --git a/maintainer/conda/environment.yml b/maintainer/conda/environment.yml index 6eadfc9ffee..5da06a6868e 100644 --- a/maintainer/conda/environment.yml +++ b/maintainer/conda/environment.yml @@ -30,7 +30,7 @@ dependencies: - sphinxcontrib-bibtex - mdaencore - waterdynamics - - imdclient + # - imdclient - pip: - mdahole2 - pathsimanalysis From a25ff7ad805558c0dafbf6350a8b867120735b24 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Sat, 28 Jun 2025 15:33:22 -0700 Subject: [PATCH 020/114] check imd file input is a string --- package/MDAnalysis/coordinates/IMD.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index e676d9a240e..a083ee629a0 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -229,6 +229,8 @@ def close(self): # NOTE: think of other edge cases as well- should be robust def parse_host_port(filename): + if not isinstance(filename, str): + raise ValueError("IMDReader: filename must be a string") if not filename.startswith("imd://"): raise ValueError( "IMDReader: URL must be in the format 'imd://host:port'" From 7ae4b21dc9daf98a0d816ae1fee0a5e6d6e6262d Mon Sep 17 00:00:00 2001 From: hmacdope Date: Sun, 29 Jun 2025 11:08:51 +1000 Subject: [PATCH 021/114] add some port parsing test --- .../MDAnalysisTests/coordinates/test_imd.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index db9d8b8afcd..411b43248bf 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -16,7 +16,7 @@ from MDAnalysis.transformations import translate import MDAnalysis as mda -from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT +from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT, parse_host_port if HAS_IMDCLIENT: import imdclient @@ -396,3 +396,37 @@ def test_n_atoms_mismatch(): f"imd://localhost:{port}", n_atoms=universe.trajectory.n_atoms + 1, ) + + + +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") +def test_n_atoms_not_specified(): + universe = mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) + port = get_free_port() + server = InThreadIMDServer(universe.trajectory) + server.set_imdsessioninfo(create_default_imdsinfo_v3()) + server.handshake_sequence("localhost", port, first_frame=True) + with pytest.raises( + ValueError, + match="IMDReader: n_atoms must be specified", + ): + IMDReader( + f"imd://localhost:{port}", + + ) + + +def test_parse_host_port(): + # Test with a valid host and port + host, port = parse_host_port("imd://localhost:8889") + assert host == "localhost" + assert port == 8889 + + # Test with a valid host and invalid port + with pytest.raises(ValueError, match="IMDReader: Port must be an integer"): + host, port = parse_host_port("imd://localhost:abcd") + + + with pytest.raises(ValueError, match="IMDReader: URL must be in the format 'imd://host:port'"): + host, port = parse_host_port("imd://localhost:blah:bleh") + From 577f7852ab1b46b6c4305fab622551305ba88990 Mon Sep 17 00:00:00 2001 From: hmacdope Date: Sun, 29 Jun 2025 11:19:20 +1000 Subject: [PATCH 022/114] add test for transformation --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 411b43248bf..fa9856d8a65 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -209,6 +209,12 @@ def test_first_dimensions(self, ref, reader): atol=1.5 * 10 ** (-ref.prec), ) + def test_transformed(self, ref, transformed): + # see transformed fixture + ref_trans = ref.first_frame.positions + 1 + ref_trans[:, 2] += 0.33 + assert_allclose(transformed.ts.positions, ref_trans) + def test_volume(self, ref, reader): # don't rewind here as in inherited base test vol = reader.ts.volume From 93cdb22ae1c3d7db87745178ba46ab37ffce0849 Mon Sep 17 00:00:00 2001 From: hmacdope Date: Sun, 29 Jun 2025 12:13:21 +1000 Subject: [PATCH 023/114] try cut buffer by 10 --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index fa9856d8a65..13d2c1dc9f6 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -323,6 +323,7 @@ def reader(self, universe, imdsinfo, port): reader = IMDReader( f"imd://localhost:{port}", n_atoms=universe.trajectory.n_atoms, + buffer_size=1*1024*1024 ) server.send_frames(1, 5) From 7e5bcb06980a9170361bbf7f9bb3c545a1126d28 Mon Sep 17 00:00:00 2001 From: hmacdope Date: Mon, 30 Jun 2025 09:51:03 +1000 Subject: [PATCH 024/114] add buffer sizes --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 13d2c1dc9f6..e0b483f272d 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -137,7 +137,7 @@ def reader(ref): # to then send handshake & first frame ref.server.handshake_sequence("localhost", ref.port) # This will connect to the test IMD Server and read the first frame - reader = ref.reader(ref.trajectory, n_atoms=ref.n_atoms) + reader = ref.reader(ref.trajectory, n_atoms=ref.n_atoms, buffer_size=1*1024*1024) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) @@ -164,7 +164,7 @@ def transformed(ref): # to then send handshake & first frame ref.server.handshake_sequence("localhost", ref.port) # This will connect to the test IMD Server and read the first frame - transformed = ref.reader(ref.trajectory, n_atoms=ref.n_atoms) + transformed = ref.reader(ref.trajectory, n_atoms=ref.n_atoms, buffer_size=1*1024*1024) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) transformed.add_transformations( From 103278b74ba207a7c9e0284b513bfac53e49dcac Mon Sep 17 00:00:00 2001 From: Lawson Woods Date: Sun, 29 Jun 2025 18:14:04 -0700 Subject: [PATCH 025/114] fail out on non-v3 --- package/MDAnalysis/coordinates/IMD.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index a083ee629a0..78e379a38aa 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -158,7 +158,11 @@ def __init__( self._imdclient = IMDClient(host, port, n_atoms, **kwargs) imdsinfo = self._imdclient.get_imdsessioninfo() - # NOTE: after testing phase, fail out on IMDv2 + if imdsinfo.version != 3: + raise NotImplementedError( + f"IMDReader: Detected IMD version v{imdsinfo.version}, " + + "but IMDReader is only compatible with v3" + ) self.ts = self._Timestep( self.n_atoms, From 5501df6b41a043a62099898e177d505be1455577 Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Tue, 1 Jul 2025 12:48:05 +1000 Subject: [PATCH 026/114] Change to `r` only --- package/MDAnalysis/coordinates/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/__init__.py b/package/MDAnalysis/coordinates/__init__.py index 8a791598ace..ad1e7fd062d 100644 --- a/package/MDAnalysis/coordinates/__init__.py +++ b/package/MDAnalysis/coordinates/__init__.py @@ -274,7 +274,7 @@ class can choose an appropriate reader automatically. | library | | | file formats`_ and | | | | | :mod:`MDAnalysis.coordinates.chemfiles` | +---------------+-----------+-------+------------------------------------------------------+ - | IMD | IP address| r/w | Receive simulation trajectory data using interactive | + | IMD | IP address| r | Receive simulation trajectory data using interactive | | | and port | | molecular dynamics version 3 (IMDv3) by configuring | | | number | | a socket address to a NAMD, GROMACS, or LAMMPS | | | | | simulation. | From a9eab4399755f2bbd1d075abf70beac85224191c Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Mon, 7 Jul 2025 11:50:04 -0700 Subject: [PATCH 027/114] remove parse port --- package/MDAnalysis/coordinates/IMD.py | 42 +++++++------------ .../MDAnalysisTests/coordinates/test_imd.py | 18 +------- 2 files changed, 15 insertions(+), 45 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 78e379a38aa..e40dac78eed 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -78,6 +78,7 @@ try: import imdclient from imdclient.IMDClient import IMDClient + from imdclient.utils import parse_host_port except ImportError: HAS_IMDCLIENT = False imdclient_version = Version("0.0.0") @@ -152,7 +153,12 @@ def __init__( raise ValueError("IMDReader: n_atoms must be specified") self.n_atoms = n_atoms - host, port = parse_host_port(filename) + if not isinstance(filename, str): + raise TypeError( + "IMDReader: filename must be a string of the form 'imd://host:port'" + ) + else: + host, port = parse_host_port(filename) # This starts the simulation self._imdclient = IMDClient(host, port, n_atoms, **kwargs) @@ -200,9 +206,7 @@ def _load_imdframe_into_ts(self, imdf): self.ts.data["dt"] = imdf.dt self.ts.data["step"] = imdf.step if imdf.energies is not None: - self.ts.data.update( - {k: v for k, v in imdf.energies.items() if k != "step"} - ) + self.ts.data.update({k: v for k, v in imdf.energies.items() if k != "step"}) if imdf.box is not None: self.ts.dimensions = core.triclinic_box(*imdf.box) if imdf.positions is not None: @@ -216,11 +220,16 @@ def _load_imdframe_into_ts(self, imdf): @staticmethod def _format_hint(thing): + if not HAS_IMDCLIENT: + return False try: + # NOTE: maybe this check should be done in parse_host_port? + if not isinstance(thing, str): + return False parse_host_port(thing) except ValueError: return False - return HAS_IMDCLIENT and True + return True def close(self): """Gracefully shut down the reader. Stops the producer thread.""" @@ -229,26 +238,3 @@ def close(self): self._imdclient.stop() # NOTE: removeme after testing logger.debug("IMDReader shut down gracefully.") - - -# NOTE: think of other edge cases as well- should be robust -def parse_host_port(filename): - if not isinstance(filename, str): - raise ValueError("IMDReader: filename must be a string") - if not filename.startswith("imd://"): - raise ValueError( - "IMDReader: URL must be in the format 'imd://host:port'" - ) - # Check if the format is correct - parts = filename.split("imd://")[1].split(":") - if len(parts) == 2: - host = parts[0] - try: - port = int(parts[1]) - return (host, port) - except ValueError as e: - raise ValueError("IMDReader: Port must be an integer") from e - else: - raise ValueError( - "IMDReader: URL must be in the format 'imd://host:port'" - ) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index e0b483f272d..32ebefb1001 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -420,20 +420,4 @@ def test_n_atoms_not_specified(): IMDReader( f"imd://localhost:{port}", - ) - - -def test_parse_host_port(): - # Test with a valid host and port - host, port = parse_host_port("imd://localhost:8889") - assert host == "localhost" - assert port == 8889 - - # Test with a valid host and invalid port - with pytest.raises(ValueError, match="IMDReader: Port must be an integer"): - host, port = parse_host_port("imd://localhost:abcd") - - - with pytest.raises(ValueError, match="IMDReader: URL must be in the format 'imd://host:port'"): - host, port = parse_host_port("imd://localhost:blah:bleh") - + ) \ No newline at end of file From 97d063622183d56fbecfd1a3e9a4e917891d2107 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Mon, 7 Jul 2025 12:01:51 -0700 Subject: [PATCH 028/114] remove parse port in test --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 32ebefb1001..aa641cec179 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -16,7 +16,7 @@ from MDAnalysis.transformations import translate import MDAnalysis as mda -from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT, parse_host_port +from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT if HAS_IMDCLIENT: import imdclient From b2239bc98bca8c0e845884bd1379b8db4ec9edb7 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Mon, 7 Jul 2025 13:41:39 -0700 Subject: [PATCH 029/114] only import parse port when imdclient exists --- package/MDAnalysis/coordinates/IMD.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index e40dac78eed..0d7914bb022 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -78,7 +78,6 @@ try: import imdclient from imdclient.IMDClient import IMDClient - from imdclient.utils import parse_host_port except ImportError: HAS_IMDCLIENT = False imdclient_version = Version("0.0.0") @@ -106,6 +105,8 @@ class MockIMDClient: category=RuntimeWarning, ) HAS_IMDCLIENT = False + # only import parse_host_port if we have imdclient + from imdclient.utils import parse_host_port logger = logging.getLogger("MDAnalysis.coordinates.IMDReader") @@ -206,7 +207,9 @@ def _load_imdframe_into_ts(self, imdf): self.ts.data["dt"] = imdf.dt self.ts.data["step"] = imdf.step if imdf.energies is not None: - self.ts.data.update({k: v for k, v in imdf.energies.items() if k != "step"}) + self.ts.data.update( + {k: v for k, v in imdf.energies.items() if k != "step"} + ) if imdf.box is not None: self.ts.dimensions = core.triclinic_box(*imdf.box) if imdf.positions is not None: @@ -226,7 +229,7 @@ def _format_hint(thing): # NOTE: maybe this check should be done in parse_host_port? if not isinstance(thing, str): return False - parse_host_port(thing) + parse_host_port(thing) # type: ignore except ValueError: return False return True From 5932f660a47cb936b87f748d0efded0ceebcbc99 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Tue, 8 Jul 2025 21:35:08 -0700 Subject: [PATCH 030/114] use monkeypatch to avoid mess sys module --- package/MDAnalysis/coordinates/IMD.py | 10 +-- .../MDAnalysisTests/coordinates/test_imd.py | 62 ++++++++++--------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 0d7914bb022..0151b0d39fc 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -78,6 +78,7 @@ try: import imdclient from imdclient.IMDClient import IMDClient + from imdclient.utils import parse_host_port except ImportError: HAS_IMDCLIENT = False imdclient_version = Version("0.0.0") @@ -105,8 +106,6 @@ class MockIMDClient: category=RuntimeWarning, ) HAS_IMDCLIENT = False - # only import parse_host_port if we have imdclient - from imdclient.utils import parse_host_port logger = logging.getLogger("MDAnalysis.coordinates.IMDReader") @@ -154,12 +153,7 @@ def __init__( raise ValueError("IMDReader: n_atoms must be specified") self.n_atoms = n_atoms - if not isinstance(filename, str): - raise TypeError( - "IMDReader: filename must be a string of the form 'imd://host:port'" - ) - else: - host, port = parse_host_port(filename) + host, port = parse_host_port(filename) # This starts the simulation self._imdclient = IMDClient(host, port, n_atoms, **kwargs) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index aa641cec179..26ea7240f6a 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -2,6 +2,7 @@ """ import sys +import importlib import pytest import pickle from types import ModuleType @@ -40,46 +41,47 @@ assert_timestep_almost_equal, ) +def test_HAS_IMDCLIENT_version(monkeypatch): + backup = sys.modules.copy() -def test_HAS_IMDCLIENT_too_old(): - # mock a version of imdclient that is too old - module_name = "imdclient" + try: + module_name = "imdclient" - sys.modules.pop(module_name, None) - sys.modules.pop("MDAnalysis.coordinates.IMD", None) - - mocked_module = ModuleType(module_name) - # too old version - mocked_module.__version__ = "0.1.0" - sys.modules[module_name] = mocked_module - - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - - assert not HAS_IMDCLIENT + # Create mock modules + mocked_module = ModuleType(module_name) + IMDClient_module = ModuleType(f"{module_name}.IMDClient") + class MockIMDClient: + pass -def test_HAS_IMDCLIENT_new_enough(): - module_name = "imdclient" - sys.modules.pop(module_name, None) - sys.modules.pop("MDAnalysis.coordinates.IMD", None) + IMDClient_module.IMDClient = MockIMDClient + mocked_module.IMDClient = IMDClient_module + mocked_module.__version__ = "0.1.4" - mocked_module = ModuleType(module_name) - IMDClient_module = ModuleType(f"{module_name}.IMDClient") + utils_module = ModuleType(f"{module_name}.utils") + utils_module.parse_host_port = lambda x: ("localhost", 12345) + mocked_module.utils = utils_module - class MockIMDClient: - pass + monkeypatch.setitem(sys.modules, module_name, mocked_module) + monkeypatch.setitem(sys.modules, f"{module_name}.IMDClient", IMDClient_module) + monkeypatch.setitem(sys.modules, f"{module_name}.utils", utils_module) - IMDClient_module.IMDClient = MockIMDClient - mocked_module.IMDClient = IMDClient_module - # new enough version - mocked_module.__version__ = "0.1.4" + sys.modules.pop("MDAnalysis.coordinates.IMD", None) + import MDAnalysis.coordinates.IMD + importlib.reload(MDAnalysis.coordinates.IMD) - sys.modules[module_name] = mocked_module - sys.modules[f"{module_name}.IMDClient"] = IMDClient_module + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + assert HAS_IMDCLIENT - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + mocked_module.__version__ = "0.0.0" + importlib.reload(MDAnalysis.coordinates.IMD) + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + assert not HAS_IMDCLIENT - assert HAS_IMDCLIENT + finally: + # Restore sys.modules to avoid side effects on other tests + sys.modules.clear() + sys.modules.update(backup) @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") From 808b998742b7a448ff0248beba7533421ef35944 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Tue, 8 Jul 2025 21:35:30 -0700 Subject: [PATCH 031/114] black imd test --- .../MDAnalysisTests/coordinates/test_imd.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 26ea7240f6a..c3aefdfbea6 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -41,6 +41,7 @@ assert_timestep_almost_equal, ) + def test_HAS_IMDCLIENT_version(monkeypatch): backup = sys.modules.copy() @@ -63,19 +64,24 @@ class MockIMDClient: mocked_module.utils = utils_module monkeypatch.setitem(sys.modules, module_name, mocked_module) - monkeypatch.setitem(sys.modules, f"{module_name}.IMDClient", IMDClient_module) + monkeypatch.setitem( + sys.modules, f"{module_name}.IMDClient", IMDClient_module + ) monkeypatch.setitem(sys.modules, f"{module_name}.utils", utils_module) sys.modules.pop("MDAnalysis.coordinates.IMD", None) import MDAnalysis.coordinates.IMD + importlib.reload(MDAnalysis.coordinates.IMD) from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + assert HAS_IMDCLIENT mocked_module.__version__ = "0.0.0" importlib.reload(MDAnalysis.coordinates.IMD) from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + assert not HAS_IMDCLIENT finally: @@ -139,7 +145,9 @@ def reader(ref): # to then send handshake & first frame ref.server.handshake_sequence("localhost", ref.port) # This will connect to the test IMD Server and read the first frame - reader = ref.reader(ref.trajectory, n_atoms=ref.n_atoms, buffer_size=1*1024*1024) + reader = ref.reader( + ref.trajectory, n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 + ) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) @@ -166,7 +174,9 @@ def transformed(ref): # to then send handshake & first frame ref.server.handshake_sequence("localhost", ref.port) # This will connect to the test IMD Server and read the first frame - transformed = ref.reader(ref.trajectory, n_atoms=ref.n_atoms, buffer_size=1*1024*1024) + transformed = ref.reader( + ref.trajectory, n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 + ) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) transformed.add_transformations( @@ -214,7 +224,7 @@ def test_first_dimensions(self, ref, reader): def test_transformed(self, ref, transformed): # see transformed fixture ref_trans = ref.first_frame.positions + 1 - ref_trans[:, 2] += 0.33 + ref_trans[:, 2] += 0.33 assert_allclose(transformed.ts.positions, ref_trans) def test_volume(self, ref, reader): @@ -325,7 +335,7 @@ def reader(self, universe, imdsinfo, port): reader = IMDReader( f"imd://localhost:{port}", n_atoms=universe.trajectory.n_atoms, - buffer_size=1*1024*1024 + buffer_size=1 * 1024 * 1024, ) server.send_frames(1, 5) @@ -407,7 +417,6 @@ def test_n_atoms_mismatch(): ) - @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") def test_n_atoms_not_specified(): universe = mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) @@ -421,5 +430,4 @@ def test_n_atoms_not_specified(): ): IMDReader( f"imd://localhost:{port}", - - ) \ No newline at end of file + ) From 2d95ac73ede9c1c3d5332d74409e71a74b6c7539 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Tue, 8 Jul 2025 23:01:44 -0700 Subject: [PATCH 032/114] add coverage test --- package/MDAnalysis/coordinates/IMD.py | 2 +- .../MDAnalysisTests/coordinates/test_imd.py | 86 ++++++++++++++----- 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 0151b0d39fc..fbaecef340d 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -176,7 +176,7 @@ def __init__( self._frame = -1 try: - self._read_next_timestep() + self.next() except StopIteration as e: raise RuntimeError("IMDReader: No data found in stream") from e diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index c3aefdfbea6..43d8def5935 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -42,7 +42,7 @@ ) -def test_HAS_IMDCLIENT_version(monkeypatch): +def test_IMDCLIENT_import(monkeypatch): backup = sys.modules.copy() try: @@ -80,10 +80,18 @@ class MockIMDClient: mocked_module.__version__ = "0.0.0" importlib.reload(MDAnalysis.coordinates.IMD) - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT, IMDReader assert not HAS_IMDCLIENT + # test initialization error + with pytest.raises( + ImportError, + match="IMDReader requires the imdclient" + ): + IMDReader("imd://localhost:12345", n_atoms=5) + + finally: # Restore sys.modules to avoid side effects on other tests sys.modules.clear() @@ -165,7 +173,8 @@ def reader(ref): initial_time=0, time_selector=None, ) - return reader + yield reader + reader.close() @staticmethod @pytest.fixture() @@ -312,21 +321,23 @@ def test_frame_collect_all_same(self, reader): pytest.skip("IMDReader has independent coordinates") -@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") -class TestStreamIteration: +@pytest.fixture +def universe(): + return mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) - @pytest.fixture - def port(self): - return get_free_port() - @pytest.fixture - def universe(self): - return mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) +@pytest.fixture +def port(): + return get_free_port() - @pytest.fixture - def imdsinfo(self): - return create_default_imdsinfo_v3() +@pytest.fixture +def imdsinfo(): + return create_default_imdsinfo_v3() + + +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") +class TestStreamIteration: @pytest.fixture def reader(self, universe, imdsinfo, port): server = InThreadIMDServer(universe.trajectory) @@ -401,11 +412,11 @@ def test_timeseries_raises(self, reader): @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") -def test_n_atoms_mismatch(): - universe = mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) +def test_n_atoms_mismatch(universe, imdsinfo, port): port = get_free_port() + imdsinfo = create_default_imdsinfo_v3() server = InThreadIMDServer(universe.trajectory) - server.set_imdsessioninfo(create_default_imdsinfo_v3()) + server.set_imdsessioninfo(imdsinfo) server.handshake_sequence("localhost", port, first_frame=True) with pytest.raises( EOFError, @@ -415,14 +426,15 @@ def test_n_atoms_mismatch(): f"imd://localhost:{port}", n_atoms=universe.trajectory.n_atoms + 1, ) + server.send_frames(1, 5) + + server.cleanup() @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") -def test_n_atoms_not_specified(): - universe = mda.Universe(COORDINATES_TOPOLOGY, COORDINATES_H5MD) - port = get_free_port() +def test_n_atoms_not_specified(universe, imdsinfo, port): server = InThreadIMDServer(universe.trajectory) - server.set_imdsessioninfo(create_default_imdsinfo_v3()) + server.set_imdsessioninfo(imdsinfo) server.handshake_sequence("localhost", port, first_frame=True) with pytest.raises( ValueError, @@ -431,3 +443,35 @@ def test_n_atoms_not_specified(): IMDReader( f"imd://localhost:{port}", ) + server.cleanup() + + +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") +def test_imd_stream_empty(universe, imdsinfo, port): + server = InThreadIMDServer(universe.trajectory) + server.set_imdsessioninfo(imdsinfo) + server.handshake_sequence("localhost", port, first_frame=True) + + with pytest.raises( + RuntimeError, + match="IMDReader: No data found in stream", + ): + IMDReader( + f"imd://localhost:{port}", + n_atoms=universe.trajectory.n_atoms, + ) + server.cleanup() + + +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") +def test_universe_format_hint(universe, imdsinfo, port): + server = InThreadIMDServer(universe.trajectory) + server.set_imdsessioninfo(imdsinfo) + server.handshake_sequence("localhost", port, first_frame=True) + u_imd = mda.Universe( + COORDINATES_TOPOLOGY, + f"imd://localhost:{port}", + n_atoms=universe.trajectory.n_atoms, + ) + assert isinstance(u_imd.trajectory, IMDReader) + server.cleanup() \ No newline at end of file From 57948c6c252d557b82b8fbd436db6df583953219 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Tue, 8 Jul 2025 23:42:47 -0700 Subject: [PATCH 033/114] add tests fix --- package/MDAnalysis/coordinates/IMD.py | 2 -- .../MDAnalysisTests/coordinates/test_imd.py | 26 ++++++++----------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index fbaecef340d..a55bab927b1 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -217,8 +217,6 @@ def _load_imdframe_into_ts(self, imdf): @staticmethod def _format_hint(thing): - if not HAS_IMDCLIENT: - return False try: # NOTE: maybe this check should be done in parse_host_port? if not isinstance(thing, str): diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 43d8def5935..89900829c5a 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -70,28 +70,28 @@ class MockIMDClient: monkeypatch.setitem(sys.modules, f"{module_name}.utils", utils_module) sys.modules.pop("MDAnalysis.coordinates.IMD", None) + + # check if imdclient is new enough import MDAnalysis.coordinates.IMD importlib.reload(MDAnalysis.coordinates.IMD) - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT assert HAS_IMDCLIENT + # check if imdclient version is too old mocked_module.__version__ = "0.0.0" importlib.reload(MDAnalysis.coordinates.IMD) - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT, IMDReader + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + from MDAnalysis.coordinates.IMD import IMDReader as IMDReader_NOClient assert not HAS_IMDCLIENT # test initialization error with pytest.raises( - ImportError, - match="IMDReader requires the imdclient" + ImportError, match="IMDReader requires the imdclient" ): - IMDReader("imd://localhost:12345", n_atoms=5) - - + IMDReader_NOClient("imd://localhost:12345", n_atoms=5) finally: # Restore sys.modules to avoid side effects on other tests sys.modules.clear() @@ -352,6 +352,7 @@ def reader(self, universe, imdsinfo, port): yield reader server.cleanup() + reader.close() def test_iterate_step(self, reader, universe): i = 0 @@ -413,8 +414,6 @@ def test_timeseries_raises(self, reader): @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") def test_n_atoms_mismatch(universe, imdsinfo, port): - port = get_free_port() - imdsinfo = create_default_imdsinfo_v3() server = InThreadIMDServer(universe.trajectory) server.set_imdsessioninfo(imdsinfo) server.handshake_sequence("localhost", port, first_frame=True) @@ -426,8 +425,6 @@ def test_n_atoms_mismatch(universe, imdsinfo, port): f"imd://localhost:{port}", n_atoms=universe.trajectory.n_atoms + 1, ) - server.send_frames(1, 5) - server.cleanup() @@ -450,8 +447,7 @@ def test_n_atoms_not_specified(universe, imdsinfo, port): def test_imd_stream_empty(universe, imdsinfo, port): server = InThreadIMDServer(universe.trajectory) server.set_imdsessioninfo(imdsinfo) - server.handshake_sequence("localhost", port, first_frame=True) - + server.handshake_sequence("localhost", port, first_frame=False) with pytest.raises( RuntimeError, match="IMDReader: No data found in stream", @@ -473,5 +469,5 @@ def test_universe_format_hint(universe, imdsinfo, port): f"imd://localhost:{port}", n_atoms=universe.trajectory.n_atoms, ) - assert isinstance(u_imd.trajectory, IMDReader) - server.cleanup() \ No newline at end of file + assert type(u_imd.trajectory).__name__ == "IMDReader" + server.cleanup() From c40a8298af2040800f2946a102cc4d2f856a05ce Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Wed, 9 Jul 2025 12:39:26 -0700 Subject: [PATCH 034/114] disable test on imd version --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 89900829c5a..ad10e4bec41 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -41,7 +41,7 @@ assert_timestep_almost_equal, ) - +@pytest.mark.skip(reason="The test interferes with sys.modules and can cause side effects") def test_IMDCLIENT_import(monkeypatch): backup = sys.modules.copy() From 05f58e538bcc0b020c1e3f84606d810759ecf6eb Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Thu, 10 Jul 2025 13:41:47 -0700 Subject: [PATCH 035/114] imd type hint --- package/MDAnalysis/coordinates/IMD.py | 18 +++++++++------- .../MDAnalysisTests/coordinates/test_imd.py | 21 +++++++++++++++++-- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index a55bab927b1..b343b81082e 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -153,7 +153,10 @@ def __init__( raise ValueError("IMDReader: n_atoms must be specified") self.n_atoms = n_atoms - host, port = parse_host_port(filename) + try: + host, port = parse_host_port(filename) + except ValueError as e: + raise ValueError(f"IMDReader: Invalid IMD URL '{filename}': {e}") # This starts the simulation self._imdclient = IMDClient(host, port, n_atoms, **kwargs) @@ -217,14 +220,13 @@ def _load_imdframe_into_ts(self, imdf): @staticmethod def _format_hint(thing): - try: - # NOTE: maybe this check should be done in parse_host_port? - if not isinstance(thing, str): - return False - parse_host_port(thing) # type: ignore - except ValueError: + if not isinstance(thing, str): + return False + # a weaker check for type hint + if thing.startswith("imd://"): + return True + else: return False - return True def close(self): """Gracefully shut down the reader. Stops the producer thread.""" diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index ad10e4bec41..c2b65954b8e 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -41,7 +41,10 @@ assert_timestep_almost_equal, ) -@pytest.mark.skip(reason="The test interferes with sys.modules and can cause side effects") + +@pytest.mark.skip( + reason="The test interferes with sys.modules and can cause side effects" +) def test_IMDCLIENT_import(monkeypatch): backup = sys.modules.copy() @@ -460,7 +463,7 @@ def test_imd_stream_empty(universe, imdsinfo, port): @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") -def test_universe_format_hint(universe, imdsinfo, port): +def test_create_imd_universe(universe, imdsinfo, port): server = InThreadIMDServer(universe.trajectory) server.set_imdsessioninfo(imdsinfo) server.handshake_sequence("localhost", port, first_frame=True) @@ -470,4 +473,18 @@ def test_universe_format_hint(universe, imdsinfo, port): n_atoms=universe.trajectory.n_atoms, ) assert type(u_imd.trajectory).__name__ == "IMDReader" + with pytest.raises(ValueError, match="IMDReader: Invalid IMD URL"): + u_imd = mda.Universe( + COORDINATES_TOPOLOGY, + f"imd://localhost:{port}/invalid", + n_atoms=universe.trajectory.n_atoms, + ) server.cleanup() + + +def test_imd_format_hint(): + assert IMDReader._format_hint("imd://localhost:12345") + assert IMDReader._format_hint("imd://localhost:12345/invalid") + assert not IMDReader._format_hint("not_a_valid_imd_url") + assert not IMDReader._format_hint(12345) + assert not IMDReader._format_hint(None) From b181cb9b53a9283523bb83418b0b7ad6265af2f3 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Thu, 10 Jul 2025 14:11:08 -0700 Subject: [PATCH 036/114] test error on mda not imdclient --- package/MDAnalysis/coordinates/IMD.py | 6 +++--- testsuite/MDAnalysisTests/coordinates/test_imd.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index b343b81082e..cd70b97a5aa 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -179,9 +179,9 @@ def __init__( self._frame = -1 try: - self.next() - except StopIteration as e: - raise RuntimeError("IMDReader: No data found in stream") from e + self._read_next_timestep() + except EOFError as e: + raise RuntimeError(f"IMDReader: Read error: {e}") from e def _read_frame(self, frame): diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index c2b65954b8e..aa78db5a397 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -421,8 +421,8 @@ def test_n_atoms_mismatch(universe, imdsinfo, port): server.set_imdsessioninfo(imdsinfo) server.handshake_sequence("localhost", port, first_frame=True) with pytest.raises( - EOFError, - match="IMDProducer: Expected n_atoms value 6, got 5. Ensure you are using the correct topology file.", + RuntimeError, + match="IMDReader: Read error", ): IMDReader( f"imd://localhost:{port}", @@ -453,7 +453,7 @@ def test_imd_stream_empty(universe, imdsinfo, port): server.handshake_sequence("localhost", port, first_frame=False) with pytest.raises( RuntimeError, - match="IMDReader: No data found in stream", + match="IMDReader: Read error", ): IMDReader( f"imd://localhost:{port}", From 2199882fabfab26d35ef6396b77afd7a9e784841 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Thu, 10 Jul 2025 14:17:59 -0700 Subject: [PATCH 037/114] close ref port properly --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index aa78db5a397..0f1a002ed45 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -147,7 +147,9 @@ class TestIMDReaderBaseAPI(MultiframeReaderTest): @pytest.fixture(scope="function") def ref(self): """Not a static method like in base class- need new server for each test""" - return IMDReference() + reference = IMDReference() + yield reference + reference.server.cleanup() @staticmethod @pytest.fixture() From 10d260d611d57826f3435bfe0af478207295a0b3 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Sat, 12 Jul 2025 23:57:57 -0700 Subject: [PATCH 038/114] test imd import in different file --- .../MDAnalysisTests/coordinates/test_imd.py | 59 --------------- .../coordinates/test_imd_import.py | 75 +++++++++++++++++++ 2 files changed, 75 insertions(+), 59 deletions(-) create mode 100644 testsuite/MDAnalysisTests/coordinates/test_imd_import.py diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 0f1a002ed45..8fba6c370e6 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -42,65 +42,6 @@ ) -@pytest.mark.skip( - reason="The test interferes with sys.modules and can cause side effects" -) -def test_IMDCLIENT_import(monkeypatch): - backup = sys.modules.copy() - - try: - module_name = "imdclient" - - # Create mock modules - mocked_module = ModuleType(module_name) - IMDClient_module = ModuleType(f"{module_name}.IMDClient") - - class MockIMDClient: - pass - - IMDClient_module.IMDClient = MockIMDClient - mocked_module.IMDClient = IMDClient_module - mocked_module.__version__ = "0.1.4" - - utils_module = ModuleType(f"{module_name}.utils") - utils_module.parse_host_port = lambda x: ("localhost", 12345) - mocked_module.utils = utils_module - - monkeypatch.setitem(sys.modules, module_name, mocked_module) - monkeypatch.setitem( - sys.modules, f"{module_name}.IMDClient", IMDClient_module - ) - monkeypatch.setitem(sys.modules, f"{module_name}.utils", utils_module) - - sys.modules.pop("MDAnalysis.coordinates.IMD", None) - - # check if imdclient is new enough - import MDAnalysis.coordinates.IMD - - importlib.reload(MDAnalysis.coordinates.IMD) - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - - assert HAS_IMDCLIENT - - # check if imdclient version is too old - mocked_module.__version__ = "0.0.0" - importlib.reload(MDAnalysis.coordinates.IMD) - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - from MDAnalysis.coordinates.IMD import IMDReader as IMDReader_NOClient - - assert not HAS_IMDCLIENT - - # test initialization error - with pytest.raises( - ImportError, match="IMDReader requires the imdclient" - ): - IMDReader_NOClient("imd://localhost:12345", n_atoms=5) - finally: - # Restore sys.modules to avoid side effects on other tests - sys.modules.clear() - sys.modules.update(backup) - - @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class IMDReference(BaseReference): def __init__(self): diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd_import.py b/testsuite/MDAnalysisTests/coordinates/test_imd_import.py new file mode 100644 index 00000000000..557ca6a27a8 --- /dev/null +++ b/testsuite/MDAnalysisTests/coordinates/test_imd_import.py @@ -0,0 +1,75 @@ +"""Test for MDAnalysis trajectory reader expectations +""" + +import sys +import importlib +import pytest +from types import ModuleType + +from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + +if HAS_IMDCLIENT: + import imdclient + from imdclient.tests.utils import ( + get_free_port, + create_default_imdsinfo_v3, + ) + from imdclient.tests.server import InThreadIMDServer + +from MDAnalysis.coordinates.IMD import IMDReader + + +def test_IMDCLIENT_import(monkeypatch): + backup = sys.modules.copy() + + try: + module_name = "imdclient" + + # Create mock modules + mocked_module = ModuleType(module_name) + IMDClient_module = ModuleType(f"{module_name}.IMDClient") + + class MockIMDClient: + pass + + IMDClient_module.IMDClient = MockIMDClient + mocked_module.IMDClient = IMDClient_module + mocked_module.__version__ = "0.1.4" + + utils_module = ModuleType(f"{module_name}.utils") + utils_module.parse_host_port = lambda x: ("localhost", 12345) + mocked_module.utils = utils_module + + monkeypatch.setitem(sys.modules, module_name, mocked_module) + monkeypatch.setitem( + sys.modules, f"{module_name}.IMDClient", IMDClient_module + ) + monkeypatch.setitem(sys.modules, f"{module_name}.utils", utils_module) + + sys.modules.pop("MDAnalysis.coordinates.IMD", None) + + # check if imdclient is new enough + import MDAnalysis.coordinates.IMD + + importlib.reload(MDAnalysis.coordinates.IMD) + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + + assert HAS_IMDCLIENT + + # check if imdclient version is too old + mocked_module.__version__ = "0.0.0" + importlib.reload(MDAnalysis.coordinates.IMD) + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + from MDAnalysis.coordinates.IMD import IMDReader as IMDReader_NOClient + + assert not HAS_IMDCLIENT + + # test initialization error + with pytest.raises( + ImportError, match="IMDReader requires the imdclient" + ): + IMDReader_NOClient("imd://localhost:12345", n_atoms=5) + finally: + # Restore sys.modules to avoid side effects on other tests + sys.modules.clear() + sys.modules.update(backup) From f89a75c830befd8b871b8a9a3bf402193aae1541 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Sat, 12 Jul 2025 23:58:08 -0700 Subject: [PATCH 039/114] add test for stream base --- .../coordinates/test_reader_api.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/testsuite/MDAnalysisTests/coordinates/test_reader_api.py b/testsuite/MDAnalysisTests/coordinates/test_reader_api.py index 4ae5c0f5c6c..2962542f511 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_reader_api.py +++ b/testsuite/MDAnalysisTests/coordinates/test_reader_api.py @@ -27,6 +27,7 @@ from MDAnalysis.coordinates.base import ( ReaderBase, SingleFrameReaderBase, + StreamReaderBase, Timestep, ) from numpy.testing import assert_allclose, assert_equal @@ -81,6 +82,24 @@ def _read_first_frame(self): self.ts.frame = 0 +class AmazingStreamReader(StreamReaderBase): + format = "AmazingStream" + + def __init__(self, filename, n_atoms): + self.n_atoms = n_atoms + self._mocked_frames = [Timestep(n_atoms) for _ in range(3)] + super().__init__(filename) + + def _read_frame(self, frame): + self._frame = frame + if self._frame >= len(self._mocked_frames): + raise EOFError("End of stream") + ts = self._mocked_frames[self._frame] + ts.frame = self._frame + self.ts = ts + return ts + + class _TestReader(object): __test__ = False """Basic API readers""" @@ -445,3 +464,82 @@ def test_iter_rewind(self, reader): assert_allclose(ts.positions, np.zeros((10, 3))) assert_allclose(reader.ts.positions, np.zeros((10, 3))) + + +class _Stream: + n_atoms = 3 + readerclass = AmazingStreamReader + + +class TestStreamReader(_Stream): + __test__ = True + + @pytest.fixture + def reader(self): + return self.readerclass("dummy", n_atoms=self.n_atoms) + + def test_repr(self, reader): + rep = repr(reader) + assert "AmazingStreamReader" in rep + assert "continuous stream" in rep + assert "3 atoms" in rep + + def test_read_and_exhaust_stream(self, reader): + ts0 = reader.next() + ts1 = reader.next() + ts2 = reader.next() + assert ts0.frame == 0 + assert ts1.frame == 1 + assert ts2.frame == 2 + + with pytest.raises(StopIteration): + reader.next() + + def test_len_and_n_frames_raise(self, reader): + with pytest.raises(RuntimeError): + _ = len(reader) + with pytest.raises(RuntimeError): + _ = reader.n_frames + + def test_rewind_raises(self, reader): + with pytest.raises(RuntimeError, match="can't be rewound"): + reader.rewind() + + def test_copy_raises(self, reader): + with pytest.raises(RuntimeError, match="does not support copying"): + reader.copy() + + def test_timeseries_raises(self, reader): + with pytest.raises(RuntimeError, match="cannot access timeseries"): + reader.timeseries() + + def test_reopen_only_once(self, reader): + reader._reopen() + with pytest.raises(RuntimeError, match="Cannot reopen stream"): + reader._reopen() + + def test_slice_reader(self, reader): + sliced = reader[slice(None, None, 2)] + with pytest.raises(RuntimeError, match="has unknown length"): + len(sliced) + with pytest.raises(RuntimeError, match="does not support indexing"): + sliced[0] + + for i, ts in enumerate(sliced): + assert ts.frame == i * 2 + + def test_check_slice_index_errors(self, reader): + with pytest.raises(ValueError, match="start.*must be None"): + reader.check_slice_indices(0, None, 1) + with pytest.raises(ValueError, match="stop.*must be None"): + reader.check_slice_indices(None, 1, 1) + with pytest.raises(ValueError, match="must be > 0"): + reader.check_slice_indices(None, None, 0) + with pytest.raises(ValueError, match="must be an integer"): + reader.check_slice_indices(None, None, 1.5) + + def test_pickle_methods(self, reader): + with pytest.raises(NotImplementedError): + reader.__getstate__() + with pytest.raises(NotImplementedError): + reader.__setstate__({}) \ No newline at end of file From 2167620de0a13b1976142fc53ef74527b66a544c Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Sun, 13 Jul 2025 00:21:11 -0700 Subject: [PATCH 040/114] move _frame to baseclass --- package/MDAnalysis/coordinates/IMD.py | 2 -- package/MDAnalysis/coordinates/base.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index cd70b97a5aa..637e66e49fa 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -176,8 +176,6 @@ def __init__( **self._ts_kwargs, ) - self._frame = -1 - try: self._read_next_timestep() except EOFError as e: diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index df4c9e03240..16fc51572ad 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1865,6 +1865,7 @@ def __init__(self, filename, convert_units=True, **kwargs): self._init_scope = True self._reopen_called = False self._first_ts = None + self._frame = -1 def _read_next_timestep(self): # No rewinding- to both load the first frame after __init__ From 831b46eb2077fa3b0dabfaccb29e9d78806ac207 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Sun, 13 Jul 2025 13:26:10 -0700 Subject: [PATCH 041/114] remove onepass --- package/MDAnalysis/coordinates/IMD.py | 1 - testsuite/MDAnalysisTests/coordinates/test_reader_api.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 637e66e49fa..c38a1f110ca 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -128,7 +128,6 @@ class IMDReader(StreamReaderBase): """ format = "IMD" - one_pass = True @store_init_arguments def __init__( diff --git a/testsuite/MDAnalysisTests/coordinates/test_reader_api.py b/testsuite/MDAnalysisTests/coordinates/test_reader_api.py index 2962542f511..b5548f0ac1e 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_reader_api.py +++ b/testsuite/MDAnalysisTests/coordinates/test_reader_api.py @@ -524,7 +524,7 @@ def test_slice_reader(self, reader): len(sliced) with pytest.raises(RuntimeError, match="does not support indexing"): sliced[0] - + for i, ts in enumerate(sliced): assert ts.frame == i * 2 @@ -542,4 +542,4 @@ def test_pickle_methods(self, reader): with pytest.raises(NotImplementedError): reader.__getstate__() with pytest.raises(NotImplementedError): - reader.__setstate__({}) \ No newline at end of file + reader.__setstate__({}) From 38f96a6e3b6e42318a29b8242da4d8a3047a95cc Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Mon, 21 Jul 2025 15:51:33 +1000 Subject: [PATCH 042/114] Update installed optional deps with new imdclient 0.2.2 package --- .github/actions/setup-deps/action.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-deps/action.yaml b/.github/actions/setup-deps/action.yaml index e7e219ba695..897dfce9107 100644 --- a/.github/actions/setup-deps/action.yaml +++ b/.github/actions/setup-deps/action.yaml @@ -82,8 +82,8 @@ inputs: default: 'seaborn>=0.7.0' tidynamics: default: 'tidynamics>=1.0.0' - # imdclient: - # default: 'imdclient' + imdclient: + default: 'imdclient>=0.2.2' # pip-installed min dependencies coverage: default: 'coverage' @@ -108,8 +108,6 @@ inputs: default: 'pyedr>=0.7.0' waterdynamics: default: 'waterdynamics' - imdclient: - default: 'imdclient==0.2.0b0' runs: using: "composite" @@ -151,6 +149,7 @@ runs: ${{ inputs.scikit-learn }} ${{ inputs.seaborn }} ${{ inputs.tidynamics }} + ${{ inputs.imdclient }} run: | # setup full variable @@ -182,7 +181,6 @@ runs: ${{ inputs.duecredit }} ${{ inputs.parmed }} ${{ inputs.pyedr }} - ${{ inputs.imdclient}} run: | # setup full variable if [ ${{ inputs.full-deps }} = "true" ]; then From e2c0913125b0de8a414b86dcde70b2819a92f148 Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Mon, 21 Jul 2025 15:52:19 +1000 Subject: [PATCH 043/114] Update azure-pipelines.yml for 0.2.2 imdclient version --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 30a1e452bab..10787d2c82c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -113,7 +113,7 @@ jobs: pytng>=0.2.3 rdkit>=2020.03.1 tidynamics>=1.0.0 - imdclient==0.2.0b0 + imdclient>=0.2.2 # remove from azure to avoid test hanging #4707 # "gsd>3.0.0" From 07756f8f095ae2c39a62eba0e28b618833696b24 Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Mon, 21 Jul 2025 15:52:46 +1000 Subject: [PATCH 044/114] Update environment.yml --- maintainer/conda/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainer/conda/environment.yml b/maintainer/conda/environment.yml index 5da06a6868e..7b69149302c 100644 --- a/maintainer/conda/environment.yml +++ b/maintainer/conda/environment.yml @@ -30,7 +30,7 @@ dependencies: - sphinxcontrib-bibtex - mdaencore - waterdynamics - # - imdclient + - imdclient>=0.2.2 - pip: - mdahole2 - pathsimanalysis From fd6175369bc835a9546669266c0eb88eb4a5092b Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Mon, 21 Jul 2025 15:53:37 +1000 Subject: [PATCH 045/114] Update IMD.py version pins for imdclient package 0.2.2 --- package/MDAnalysis/coordinates/IMD.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index c38a1f110ca..ee80acb46b9 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -73,7 +73,7 @@ from packaging.version import Version -MIN_IMDCLIENT_VERSION = Version("0.1.4") +MIN_IMDCLIENT_VERSION = Version("0.2.2") try: import imdclient @@ -97,7 +97,7 @@ class MockIMDClient: HAS_IMDCLIENT = True imdclient_version = Version(imdclient.__version__) - # Check for compatibility: currently needs to be >=0.1.4 + # Check for compatibility: currently needs to be >=0.2.2 if imdclient_version < MIN_IMDCLIENT_VERSION: warnings.warn( f"imdclient version {imdclient_version} is too old; " From db59525402f251ed8667a7781dcef4f37d0921ba Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Mon, 21 Jul 2025 15:54:42 +1000 Subject: [PATCH 046/114] Update pyproject.toml for imdclient package 0.2.2 --- package/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/pyproject.toml b/package/pyproject.toml index 10eebfdaa89..4de0fef2f90 100644 --- a/package/pyproject.toml +++ b/package/pyproject.toml @@ -76,7 +76,7 @@ extra_formats = [ "pytng>=0.2.3", "gsd>3.0.0", "rdkit>=2020.03.1", - "imdclient", + "imdclient>=0.2.2", ] analysis = [ "biopython>=1.80", From bbcb14e51409cb29e2b1e790670bd94cb85e654b Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Mon, 21 Jul 2025 16:21:18 +1000 Subject: [PATCH 047/114] Update version pin to be dynamic in test_imd_import.py --- testsuite/MDAnalysisTests/coordinates/test_imd_import.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd_import.py b/testsuite/MDAnalysisTests/coordinates/test_imd_import.py index 557ca6a27a8..85caba4f578 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd_import.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd_import.py @@ -6,7 +6,7 @@ import pytest from types import ModuleType -from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT +from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT, MIN_IMDCLIENT_VERSION if HAS_IMDCLIENT: import imdclient @@ -34,7 +34,7 @@ class MockIMDClient: IMDClient_module.IMDClient = MockIMDClient mocked_module.IMDClient = IMDClient_module - mocked_module.__version__ = "0.1.4" + mocked_module.__version__ = MIN_IMDCLIENT_VERSION utils_module = ModuleType(f"{module_name}.utils") utils_module.parse_host_port = lambda x: ("localhost", 12345) From 60c434c069fac1506a3573e0adc0112d04680ef6 Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Mon, 21 Jul 2025 16:35:06 +1000 Subject: [PATCH 048/114] Update test_imd_import.py --- testsuite/MDAnalysisTests/coordinates/test_imd_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd_import.py b/testsuite/MDAnalysisTests/coordinates/test_imd_import.py index 85caba4f578..fe2698d0cdf 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd_import.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd_import.py @@ -34,7 +34,7 @@ class MockIMDClient: IMDClient_module.IMDClient = MockIMDClient mocked_module.IMDClient = IMDClient_module - mocked_module.__version__ = MIN_IMDCLIENT_VERSION + mocked_module.__version__ = str(MIN_IMDCLIENT_VERSION) utils_module = ModuleType(f"{module_name}.utils") utils_module.parse_host_port = lambda x: ("localhost", 12345) From 3c04d37b5ece0f2279de35e1d3d14bd7826c5b70 Mon Sep 17 00:00:00 2001 From: hmacdope Date: Thu, 24 Jul 2025 14:56:28 +1000 Subject: [PATCH 049/114] make changes for new testserver API --- .../MDAnalysisTests/coordinates/test_imd.py | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 8fba6c370e6..d3afea86504 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -3,6 +3,7 @@ import sys import importlib +from weakref import ref import pytest import pickle from types import ModuleType @@ -46,7 +47,6 @@ class IMDReference(BaseReference): def __init__(self): super(IMDReference, self).__init__() - self.port = get_free_port() # Serve TRR traj data via the server traj = mda.coordinates.TRR.TRRReader(COORDINATES_TRR) self.server = InThreadIMDServer(traj) @@ -55,7 +55,7 @@ def __init__(self): self.n_atoms = traj.n_atoms self.prec = 3 - self.trajectory = f"imd://localhost:{self.port}" + self.trajectory = "imd://localhost" self.topology = COORDINATES_TOPOLOGY self.changing_dimensions = True self.reader = IMDReader @@ -97,10 +97,10 @@ def ref(self): def reader(ref): # This will start the test IMD Server, waiting for a connection # to then send handshake & first frame - ref.server.handshake_sequence("localhost", ref.port) + ref.server.handshake_sequence("localhost") # This will connect to the test IMD Server and read the first frame reader = ref.reader( - ref.trajectory, n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 + ref.trajectory + ref.server.port, n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 ) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) @@ -127,10 +127,10 @@ def reader(ref): def transformed(ref): # This will start the test IMD Server, waiting for a connection # to then send handshake & first frame - ref.server.handshake_sequence("localhost", ref.port) + ref.server.handshake_sequence("localhost") # This will connect to the test IMD Server and read the first frame transformed = ref.reader( - ref.trajectory, n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 + ref.trajectory + ref.server.port, n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 ) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) @@ -285,12 +285,12 @@ def imdsinfo(): @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class TestStreamIteration: @pytest.fixture - def reader(self, universe, imdsinfo, port): + def reader(self, universe, imdsinfo): server = InThreadIMDServer(universe.trajectory) server.set_imdsessioninfo(imdsinfo) - server.handshake_sequence("localhost", port, first_frame=True) + server.handshake_sequence("localhost", first_frame=True) reader = IMDReader( - f"imd://localhost:{port}", + f"imd://localhost:{server.port}", n_atoms=universe.trajectory.n_atoms, buffer_size=1 * 1024 * 1024, ) @@ -359,60 +359,60 @@ def test_timeseries_raises(self, reader): @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") -def test_n_atoms_mismatch(universe, imdsinfo, port): +def test_n_atoms_mismatch(universe, imdsinfo): server = InThreadIMDServer(universe.trajectory) server.set_imdsessioninfo(imdsinfo) - server.handshake_sequence("localhost", port, first_frame=True) + server.handshake_sequence("localhost", first_frame=True) with pytest.raises( RuntimeError, match="IMDReader: Read error", ): IMDReader( - f"imd://localhost:{port}", + f"imd://localhost:{server.port}", n_atoms=universe.trajectory.n_atoms + 1, ) server.cleanup() @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") -def test_n_atoms_not_specified(universe, imdsinfo, port): +def test_n_atoms_not_specified(universe, imdsinfo): server = InThreadIMDServer(universe.trajectory) server.set_imdsessioninfo(imdsinfo) - server.handshake_sequence("localhost", port, first_frame=True) + server.handshake_sequence("localhost", first_frame=True) with pytest.raises( ValueError, match="IMDReader: n_atoms must be specified", ): IMDReader( - f"imd://localhost:{port}", + f"imd://localhost:{server.port}", ) server.cleanup() @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") -def test_imd_stream_empty(universe, imdsinfo, port): +def test_imd_stream_empty(universe, imdsinfo): server = InThreadIMDServer(universe.trajectory) server.set_imdsessioninfo(imdsinfo) - server.handshake_sequence("localhost", port, first_frame=False) + server.handshake_sequence("localhost", first_frame=False) with pytest.raises( RuntimeError, match="IMDReader: Read error", ): IMDReader( - f"imd://localhost:{port}", + f"imd://localhost:{server.port}", n_atoms=universe.trajectory.n_atoms, ) server.cleanup() @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") -def test_create_imd_universe(universe, imdsinfo, port): +def test_create_imd_universe(universe, imdsinfo): server = InThreadIMDServer(universe.trajectory) server.set_imdsessioninfo(imdsinfo) - server.handshake_sequence("localhost", port, first_frame=True) + server.handshake_sequence("localhost", first_frame=True) u_imd = mda.Universe( COORDINATES_TOPOLOGY, - f"imd://localhost:{port}", + f"imd://localhost:{server.port}", n_atoms=universe.trajectory.n_atoms, ) assert type(u_imd.trajectory).__name__ == "IMDReader" From 6a9115aae10fc5fa6a2a7c63d1fcd98f35564b39 Mon Sep 17 00:00:00 2001 From: hmacdope Date: Thu, 24 Jul 2025 14:57:42 +1000 Subject: [PATCH 050/114] remove n_atoms_mismatch test --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index d3afea86504..22931919ec3 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -358,21 +358,6 @@ def test_timeseries_raises(self, reader): reader.timeseries() -@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") -def test_n_atoms_mismatch(universe, imdsinfo): - server = InThreadIMDServer(universe.trajectory) - server.set_imdsessioninfo(imdsinfo) - server.handshake_sequence("localhost", first_frame=True) - with pytest.raises( - RuntimeError, - match="IMDReader: Read error", - ): - IMDReader( - f"imd://localhost:{server.port}", - n_atoms=universe.trajectory.n_atoms + 1, - ) - server.cleanup() - @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") def test_n_atoms_not_specified(universe, imdsinfo): From 27597e7e4d246f465aaf2c1e73d05fc41b5ee4f9 Mon Sep 17 00:00:00 2001 From: hmacdope Date: Thu, 24 Jul 2025 15:53:16 +1000 Subject: [PATCH 051/114] fix str concat --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 22931919ec3..6391d7c3fd6 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -100,7 +100,7 @@ def reader(ref): ref.server.handshake_sequence("localhost") # This will connect to the test IMD Server and read the first frame reader = ref.reader( - ref.trajectory + ref.server.port, n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 + ref.trajectory + str(ref.server.port), n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 ) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) @@ -130,7 +130,7 @@ def transformed(ref): ref.server.handshake_sequence("localhost") # This will connect to the test IMD Server and read the first frame transformed = ref.reader( - ref.trajectory + ref.server.port, n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 + ref.trajectory + str(ref.server.port), n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 ) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) From d04ff968cf14eb3e57d65cfd657f32229ed25dbc Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Thu, 24 Jul 2025 19:05:18 +1000 Subject: [PATCH 052/114] Fix host port format --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 6391d7c3fd6..b279efd84ff 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -100,7 +100,7 @@ def reader(ref): ref.server.handshake_sequence("localhost") # This will connect to the test IMD Server and read the first frame reader = ref.reader( - ref.trajectory + str(ref.server.port), n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 + f"{ref.trajectory}:{ref.server.port}", n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 ) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) @@ -130,7 +130,7 @@ def transformed(ref): ref.server.handshake_sequence("localhost") # This will connect to the test IMD Server and read the first frame transformed = ref.reader( - ref.trajectory + str(ref.server.port), n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 + f"{ref.trajectory}:{ref.server.port}", n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 ) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) From 10dfe273d00c65974629b63c010a468ea5fb7379 Mon Sep 17 00:00:00 2001 From: Lawson Date: Thu, 24 Jul 2025 11:52:17 -0700 Subject: [PATCH 053/114] doc tweaks --- package/MDAnalysis/coordinates/IMD.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index ee80acb46b9..96fba8d8829 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -115,14 +115,23 @@ class IMDReader(StreamReaderBase): Reader that supports the Interactive Molecular Dynamics (IMD) protocol for reading simulation data using the IMDClient. + By using the keyword `buffer_size`, you can change the amount of memory the :class:`IMDClient` + allocates to its internal buffer. For analyses that periodically perform + some heavier computation at some fixed interval, i.e., once every 200 received frames, + increasing this value will decrease the amount of time the simulation engine spends in + a paused state and potentially decreasing total analysis time, but will require more RAM. + Parameters ---------- filename : a string of the form "imd://host:port" where host is the hostname - or IP address of the listening GROMACS server and port + or IP address of the listening simultion engine's IMD server and port is the port number. n_atoms : int (optional) number of atoms in the system. defaults to number of atoms in the topology. Don't set this unless you know what you're doing. + buffer_size: int (optional) default=10*(1024**2) + number of bytes of memory to allocate to the :class:`IMDClient`'s + internal buffer. Defaults to 10 megabytes. kwargs : dict (optional) keyword arguments passed to the constructed :class:`IMDClient` """ @@ -133,8 +142,8 @@ class IMDReader(StreamReaderBase): def __init__( self, filename, - convert_units=True, n_atoms=None, + buffer_size=10*(1012**2), **kwargs, ): if not HAS_IMDCLIENT: @@ -158,7 +167,7 @@ def __init__( raise ValueError(f"IMDReader: Invalid IMD URL '{filename}': {e}") # This starts the simulation - self._imdclient = IMDClient(host, port, n_atoms, **kwargs) + self._imdclient = IMDClient(host, port, n_atoms, buffer_size=buffer_size, **kwargs) imdsinfo = self._imdclient.get_imdsessioninfo() if imdsinfo.version != 3: From c6a9f39a8cd37ef9ed636d630a83e27a2ca62ba9 Mon Sep 17 00:00:00 2001 From: Lawson Date: Thu, 24 Jul 2025 11:54:39 -0700 Subject: [PATCH 054/114] typo --- package/MDAnalysis/coordinates/IMD.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 96fba8d8829..62047d9c739 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -119,7 +119,7 @@ class IMDReader(StreamReaderBase): allocates to its internal buffer. For analyses that periodically perform some heavier computation at some fixed interval, i.e., once every 200 received frames, increasing this value will decrease the amount of time the simulation engine spends in - a paused state and potentially decreasing total analysis time, but will require more RAM. + a paused state and potentially decrease total analysis time, but will require more RAM. Parameters ---------- From 65a1bf8ef87629e5a18c08c3711afa6e0a7f9aed Mon Sep 17 00:00:00 2001 From: Lawson Date: Thu, 24 Jul 2025 16:13:26 -0700 Subject: [PATCH 055/114] clarify need for gromacs configuration before example command --- package/MDAnalysis/coordinates/IMD.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 62047d9c739..302ec34d711 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -2,14 +2,23 @@ IMDReader --- :mod:`MDAnalysis.coordinates.IMD` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:class:`MDAnalysis.coordinates.IMD.IMDReader` is a class that implements the Interactive Molecular Dynamics (IMD) protocol for reading simulation +:class:`MDAnalysis.coordinates.IMD.IMDReader` is a class that implements the +`Interactive Molecular Dynamics (IMD) protocol `_ for reading simulation data using the IMDClient (see `imdclient `_). The protocol allows two-way communicating molecular simulation data through a socket. Via IMD, a simulation engine sends data to a receiver (in this case, the IMDClient) and the receiver can send forces and specific control -requests (such as pausing, resuming, or terminating the simulation) back to the simulation engine. It currently supports -simulation running with GROMACS, LAMMPS, or NAMD. +requests (such as pausing, resuming, or terminating the simulation) back to the simulation engine. -For example, when running a simulation with GROMACS that supports streaming, use the following commands: +IMDv3, the newest version of the protocol, is the one supported by this reader class and is implemented in GROMACS, LAMMPS, and NAMD at varying +stages of development. See the `imdclient simulation engine docs `_ for more. + +IMDv2, the first version to be broadly adopted, is currently available as a part of official releases of GROMACS, LAMMPS, and NAMD. However, +this reader class does not currently provide support for it since it was designed for visualization and gaps are allowed in the stream +(i.e., an inconsistent number of integrator time steps between transmitted coordinate arrays is allowed) + +As an example of reading a stream, after configuring GROMACS to run a simulation with IMDv3 enabled +(see the `imdclient simulation engine docs `_ for +up-to-date resources on configuring each simulation engine), use the following commands: .. code-block:: bash From b1502ff301bc304b72e083745d4bae5003f77b9c Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Fri, 25 Jul 2025 08:30:02 -0700 Subject: [PATCH 056/114] buffer docs tweak and typo --- package/MDAnalysis/coordinates/IMD.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 302ec34d711..c9104af0368 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -125,15 +125,17 @@ class IMDReader(StreamReaderBase): data using the IMDClient. By using the keyword `buffer_size`, you can change the amount of memory the :class:`IMDClient` - allocates to its internal buffer. For analyses that periodically perform - some heavier computation at some fixed interval, i.e., once every 200 received frames, - increasing this value will decrease the amount of time the simulation engine spends in - a paused state and potentially decrease total analysis time, but will require more RAM. + allocates to its internal buffer. The buffer size determines how many frames can be stored + in memory as data is received from the socket and awaits reading by the client. For analyses + that periodically perform heavier computation at fixed intervals, say for example once every + 200 received frames, increasing this value will decrease the amount of time the simulation + engine spends in a paused state and potentially decrease total analysis time, but will require + more RAM. Parameters ---------- filename : a string of the form "imd://host:port" where host is the hostname - or IP address of the listening simultion engine's IMD server and port + or IP address of the listening simulation engine's IMD server and port is the port number. n_atoms : int (optional) number of atoms in the system. defaults to number of atoms From 74050f477f8e60ab3fb67461797f2890def158a2 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Fri, 25 Jul 2025 08:32:27 -0700 Subject: [PATCH 057/114] buffer default value change --- package/MDAnalysis/coordinates/IMD.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index c9104af0368..ee6928c0fee 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -154,7 +154,7 @@ def __init__( self, filename, n_atoms=None, - buffer_size=10*(1012**2), + buffer_size=10*(1024**2), **kwargs, ): if not HAS_IMDCLIENT: From 9338d96eade452397ff981699889890babdf11a8 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Fri, 25 Jul 2025 10:01:47 -0700 Subject: [PATCH 058/114] Update: AUTHORS --- package/AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/AUTHORS b/package/AUTHORS index 8c94548ad25..77718d36209 100644 --- a/package/AUTHORS +++ b/package/AUTHORS @@ -259,7 +259,7 @@ Chronological list of authors - Tulga-Erdene Sodjargal - Gareth Elliott - Marc Schuh - + - Amruthesh Thirumalaiswamy External code ------------- From 8f154f292ee854bcc80b522be513544c7cbf3fa3 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Fri, 25 Jul 2025 10:02:03 -0700 Subject: [PATCH 059/114] Update: CHANGELOG --- package/CHANGELOG | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package/CHANGELOG b/package/CHANGELOG index ecf87e89159..6a51da10b80 100644 --- a/package/CHANGELOG +++ b/package/CHANGELOG @@ -63,6 +63,9 @@ Enhancements (PR #5038) * Moved distopia checking function to common import location in MDAnalysisTest.util (PR #5038) + * New coordinate reader: Added `IMDReader` for reading real-time streamed + molecular dynamics simulation data using the IMDv3 protocol - requires + `imdclient` package (Issue #4827, PR #4923) Changes * Refactored the RDKit converter code to move the inferring code in a separate From c9ac065b029f1017465e61304a20f1a78bc159fd Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Fri, 25 Jul 2025 10:17:40 -0700 Subject: [PATCH 060/114] black `IMD.py` --- package/MDAnalysis/coordinates/IMD.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index ee6928c0fee..425a8147f26 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -154,7 +154,7 @@ def __init__( self, filename, n_atoms=None, - buffer_size=10*(1024**2), + buffer_size=10 * (1024**2), **kwargs, ): if not HAS_IMDCLIENT: @@ -178,7 +178,9 @@ def __init__( raise ValueError(f"IMDReader: Invalid IMD URL '{filename}': {e}") # This starts the simulation - self._imdclient = IMDClient(host, port, n_atoms, buffer_size=buffer_size, **kwargs) + self._imdclient = IMDClient( + host, port, n_atoms, buffer_size=buffer_size, **kwargs + ) imdsinfo = self._imdclient.get_imdsessioninfo() if imdsinfo.version != 3: From a2b2136589941f764aca0d32ae47288167b34ec8 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Fri, 25 Jul 2025 10:20:02 -0700 Subject: [PATCH 061/114] black `test_imd.py` --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index b279efd84ff..1a229f78806 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -100,7 +100,9 @@ def reader(ref): ref.server.handshake_sequence("localhost") # This will connect to the test IMD Server and read the first frame reader = ref.reader( - f"{ref.trajectory}:{ref.server.port}", n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 + f"{ref.trajectory}:{ref.server.port}", + n_atoms=ref.n_atoms, + buffer_size=1 * 1024 * 1024, ) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) @@ -130,7 +132,9 @@ def transformed(ref): ref.server.handshake_sequence("localhost") # This will connect to the test IMD Server and read the first frame transformed = ref.reader( - f"{ref.trajectory}:{ref.server.port}", n_atoms=ref.n_atoms, buffer_size=1 * 1024 * 1024 + f"{ref.trajectory}:{ref.server.port}", + n_atoms=ref.n_atoms, + buffer_size=1 * 1024 * 1024, ) # Send the rest of the frames- small enough to all fit in socket itself ref.server.send_frames(1, 5) @@ -358,7 +362,6 @@ def test_timeseries_raises(self, reader): reader.timeseries() - @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") def test_n_atoms_not_specified(universe, imdsinfo): server = InThreadIMDServer(universe.trajectory) From 481163b7bfef647a4660d50743a86ea0f79bd927 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 07:37:13 -0700 Subject: [PATCH 062/114] test_imd.py: import reorder --- .../MDAnalysisTests/coordinates/test_imd.py | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 1a229f78806..8e63630192f 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -1,48 +1,44 @@ """Test for MDAnalysis trajectory reader expectations """ -import sys import importlib -from weakref import ref -import pytest import pickle +import sys from types import ModuleType +from weakref import ref +import pytest import numpy as np from numpy.testing import ( + assert_allclose, assert_almost_equal, assert_array_almost_equal, assert_equal, - assert_allclose, ) -from MDAnalysis.transformations import translate import MDAnalysis as mda -from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT +from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT, IMDReader +from MDAnalysis.transformations import translate if HAS_IMDCLIENT: import imdclient + from imdclient.tests.server import InThreadIMDServer from imdclient.tests.utils import ( - get_free_port, create_default_imdsinfo_v3, + get_free_port, ) - from imdclient.tests.server import InThreadIMDServer - -from MDAnalysis.coordinates.IMD import IMDReader +from MDAnalysisTests.coordinates.base import ( + assert_timestep_almost_equal, + BaseReference, + MultiframeReaderTest, +) from MDAnalysisTests.datafiles import ( + COORDINATES_H5MD, COORDINATES_TOPOLOGY, COORDINATES_TRR, - COORDINATES_H5MD, ) -from MDAnalysisTests.coordinates.base import ( - MultiframeReaderTest, - BaseReference, - assert_timestep_almost_equal, -) - - @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class IMDReference(BaseReference): def __init__(self): From b2e4bd99a140793f4d06187fa5518c4785d2b31a Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 07:47:46 -0700 Subject: [PATCH 063/114] placeholder citation removed --- .../source/documentation_pages/references.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/package/doc/sphinx/source/documentation_pages/references.rst b/package/doc/sphinx/source/documentation_pages/references.rst index 55e0b408a49..284d5a63b78 100644 --- a/package/doc/sphinx/source/documentation_pages/references.rst +++ b/package/doc/sphinx/source/documentation_pages/references.rst @@ -229,13 +229,16 @@ If you use H5MD files using pp. 18 – 26, 2021. doi:`10.25080/majora-1b6fd038-005. `_ -If you use IMD capability with :mod:`MDAnalysis.coordinates.IMD.py`, please cite -[IMDv3paper]_. +.. comment:: -.. [IMDv3paper] Authors (YEAR). - IMDv3 Manuscript Title. - *Journal*, 185. doi:`insert-doi-here - `_ + If you use IMD capability with :mod:`MDAnalysis.coordinates.IMD.py`, please cite [IMDv3paper]_. + + .. [IMDv3paper] Authors (YEAR). + IMDv3 Manuscript Title. + *Journal*, 185. doi:`insert-doi-here `_ + +.. todo:: Fill in the final IMDv3 citation once the paper is published. + See https://github.com/MDAnalysis/mdanalysis/issues/5094 .. _citations-using-duecredit: From aad39f14f74d162096a4e698918373b000da80b6 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 08:46:37 -0700 Subject: [PATCH 064/114] Correction: `_step` called instead of `step` --- package/MDAnalysis/coordinates/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 16fc51572ad..ef97ec47e23 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -2035,7 +2035,7 @@ def __next__(self): try: # Burn the timesteps until we reach the desired step # Don't use next() to avoid unnecessary transformations - while self.trajectory._frame + 1 % self.step != 0: + while (self.trajectory._frame + 1) % self._step != 0: self.trajectory._read_next_timestep() except (EOFError, IOError): # Don't rewind here like we normally would From 17f2de794484c92e6f9ad8a8b4a175c55024f289 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 08:46:56 -0700 Subject: [PATCH 065/114] Doc: `StreamFrameIteratorSliced` --- package/MDAnalysis/coordinates/base.py | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index ef97ec47e23..41c3e89a563 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -2019,6 +2019,41 @@ def __repr__(self): class StreamFrameIteratorSliced(FrameIteratorBase): """Iterator for sliced frames in a streamed trajectory. + + This iterator is created when slicing a streaming trajectory with a step + parameter (e.g., ``trajectory[::n]``). It reads every nth frame from the + continuous stream, where n is the step size, discarding intermediate frames + for performance. + + This differs from iterating over all frames (``trajectory[:]``) which uses + :class:`FrameIteratorAll` and processes every frame sequentially without + skipping. + + Streaming constraints apply: + + - Frames cannot be accessed randomly (no indexing support) + - The total number of frames is unknown until streaming ends + - Rewinding or restarting iteration is not possible + - Only forward iteration with a fixed step size is supported + + Parameters + ---------- + trajectory : StreamReaderBase + The streaming trajectory reader to iterate over. Must be a + stream-based reader that supports continuous data reading. + step : int + Step size for iteration. Must be a positive integer. A step + of 1 reads every frame, step of 2 reads every other frame, etc. + + See Also + -------- + StreamReaderBase + FrameIteratorBase + + Note + ---- + This iterator is automatically selected when using slice notation with + a step parameter on streaming trajectories. """ def __init__(self, trajectory, step): From b1b0fa79a7149eb8a6aa3fdf14f448dbd1be7549 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 09:38:59 -0700 Subject: [PATCH 066/114] Doc: `StreamReaderBase` --- package/MDAnalysis/coordinates/base.py | 31 +++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 41c3e89a563..9556ae7fbf9 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1851,9 +1851,34 @@ def convert(self, obj): class StreamReaderBase(ReaderBase): """Base class for readers that read a continuous stream of data. - This class is used for readers that read a continuous stream of data, - such as a live feed from a simulation. This places some constraints on the - reader, such as not being able to rewind or iterate more than once. + This class is designed for readers that process continuous data streams, + such as live feeds from simulations. Unlike traditional trajectory readers + that can randomly access frames, streaming readers have fundamental constraints: + + - **No random access**: Cannot seek to arbitrary frames (no ``traj[5]``) + - **No rewinding**: Cannot restart or rewind the stream + - **No length**: Total number of frames is unknown until stream ends + - **Forward-only**: Can only iterate sequentially through frames + - **No copying**: Cannot create independent copies of the reader + + The reader raises ``RuntimeError`` for operations that require random + access or rewinding, including ``rewind()``, ``copy()``, ``timeseries()``, + and ``len()``. Only slice notation is supported for iteration. + + Parameters + ---------- + filename : str or file-like + Source of the streaming data + convert_units : bool, optional + Whether to convert units from native to MDAnalysis units (default: True) + **kwargs + Additional keyword arguments passed to the parent ReaderBase + + See Also + -------- + StreamFrameIteratorSliced : Iterator for stepped streaming access + ReaderBase : Base class for standard trajectory readers + .. versionadded:: 2.10.0 """ From dcb29d98b862cca04edfb30ca1e159e7ec65cf54 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 09:39:33 -0700 Subject: [PATCH 067/114] Doc: `StreamReaderBase`'s `__getitem__` --- package/MDAnalysis/coordinates/base.py | 34 ++++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 9556ae7fbf9..3b964467d48 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1966,17 +1966,35 @@ def timeseries(self, **kwargs): ) def __getitem__(self, frame): - """Return the Timestep corresponding to *frame*. + """Return an iterator for slicing a streaming trajectory. - If `frame` is a integer then the corresponding frame is - returned. Negative numbers are counted from the end. + Parameters + ---------- + frame : slice + Slice object. Only the step parameter is meaningful for streams. - If frame is a :class:`slice` then an iterator is returned that - allows iteration over that part of the trajectory. + Returns + ------- + FrameIteratorAll or StreamFrameIteratorSliced + Iterator for the requested slice. - Note - ---- - *frame* is a 0-based frame index. + Raises + ------ + TypeError + If frame is not a slice object. + ValueError + If slice contains start or stop values. + + Examples + -------- + >>> for ts in traj[:]: # All frames sequentially + ... process(ts) + >>> for ts in traj[::5]: # Every 5th frame + ... process(ts) + + See Also + -------- + StreamFrameIteratorSliced """ if isinstance(frame, slice): _, _, step = self.check_slice_indices( From 844a37127f796fd8d1d0e6382d04d57b07014948 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 09:40:02 -0700 Subject: [PATCH 068/114] Edit: `StreamFrameIteratorSliced` docs --- package/MDAnalysis/coordinates/base.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 3b964467d48..f9f434cf741 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -2063,16 +2063,15 @@ def __repr__(self): class StreamFrameIteratorSliced(FrameIteratorBase): """Iterator for sliced frames in a streamed trajectory. - This iterator is created when slicing a streaming trajectory with a step - parameter (e.g., ``trajectory[::n]``). It reads every nth frame from the - continuous stream, where n is the step size, discarding intermediate frames - for performance. + Created when slicing a streaming trajectory with a step parameter + (e.g., ``trajectory[::n]``). Reads every nth frame from the continuous + stream, discarding intermediate frames for performance. This differs from iterating over all frames (``trajectory[:]``) which uses :class:`FrameIteratorAll` and processes every frame sequentially without skipping. - Streaming constraints apply: + Streaming constraints apply to the sliced iterator: - Frames cannot be accessed randomly (no indexing support) - The total number of frames is unknown until streaming ends @@ -2092,11 +2091,6 @@ class StreamFrameIteratorSliced(FrameIteratorBase): -------- StreamReaderBase FrameIteratorBase - - Note - ---- - This iterator is automatically selected when using slice notation with - a step parameter on streaming trajectories. """ def __init__(self, trajectory, step): From 806a0ff69f0b9955a9e2c139e868504297dc6be4 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 09:48:01 -0700 Subject: [PATCH 069/114] Doc: Update base Reader list Added to `StreamReaderBase` to list of base Readers. --- package/MDAnalysis/coordinates/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/__init__.py b/package/MDAnalysis/coordinates/__init__.py index ad1e7fd062d..9cfb08dc01d 100644 --- a/package/MDAnalysis/coordinates/__init__.py +++ b/package/MDAnalysis/coordinates/__init__.py @@ -53,7 +53,7 @@ class that defines a common :ref:`Trajectory API` and allows other code to :class:`~MDAnalysis.coordinates.base.ProtoReader` object; all Readers are accessible through this entry point in the same manner ("`duck typing`_"). -There are three types of base Reader which act as starting points for each +There are four types of base Reader which act as starting points for each specific format. These are: :class:`~MDAnalysis.coordinates.base.ReaderBase` @@ -66,6 +66,12 @@ class that defines a common :ref:`Trajectory API` and allows other code to frame of information. This is used with formats such as GRO and CRD +:class:`~MDAnalysis.coordinates.base.StreamReaderBase` + A specialized Reader for continuous data streams such as live + simulation feeds. Unlike standard readers, streaming readers cannot + randomly access frames, rewind, or determine total length. This is + used for real-time trajectory data from simulations via IMD connections. + :class:`~MDAnalysis.coordinates.chain.ChainReader` An advanced Reader designed to read a sequence of files, to provide iteration over all the frames in each file seamlessly. From f69d741a3a7c0e33013983fff90dc1d5db54a890 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 09:58:57 -0700 Subject: [PATCH 070/114] Remove: whitespace --- package/MDAnalysis/coordinates/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index f9f434cf741..ca4536a86a6 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -90,7 +90,6 @@ .. autoclass:: StreamReaderBase :members: - .. _WritersBase: From 3e81d9614cb4c29e30fabf6eb8d47bfa0a548aed Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 10:05:09 -0700 Subject: [PATCH 071/114] Update: IMD format --- package/MDAnalysis/coordinates/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package/MDAnalysis/coordinates/__init__.py b/package/MDAnalysis/coordinates/__init__.py index 9cfb08dc01d..c9da0b10fca 100644 --- a/package/MDAnalysis/coordinates/__init__.py +++ b/package/MDAnalysis/coordinates/__init__.py @@ -280,10 +280,10 @@ class can choose an appropriate reader automatically. | library | | | file formats`_ and | | | | | :mod:`MDAnalysis.coordinates.chemfiles` | +---------------+-----------+-------+------------------------------------------------------+ - | IMD | IP address| r | Receive simulation trajectory data using interactive | - | | and port | | molecular dynamics version 3 (IMDv3) by configuring | - | | number | | a socket address to a NAMD, GROMACS, or LAMMPS | - | | | | simulation. | + | IMD | imd:// | r | Receive simulation trajectory data using interactive | + | | : | | molecular dynamics version 3 (IMDv3) by configuring | + | | | | a socket address to a NAMD, GROMACS, or LAMMPS | + | | | | simulation. :mod:`MDAnalysis.coordinates.IMD` | +---------------+-----------+-------+------------------------------------------------------+ .. [#a] This format can also be used to provide basic *topology* From 230f1a6fe1d1bb4fd5bbff47336d69341e3e0279 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 22:51:07 -0700 Subject: [PATCH 072/114] Change: Error Message type --- package/MDAnalysis/coordinates/IMD.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 425a8147f26..029b351ab37 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -184,7 +184,7 @@ def __init__( imdsinfo = self._imdclient.get_imdsessioninfo() if imdsinfo.version != 3: - raise NotImplementedError( + raise ValueError( f"IMDReader: Detected IMD version v{imdsinfo.version}, " + "but IMDReader is only compatible with v3" ) From c89c0b7b799dbcd318e7e22b5623303ace9620df Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 22:52:21 -0700 Subject: [PATCH 073/114] Update: IMDReader docs --- package/MDAnalysis/coordinates/IMD.py | 38 +++++++++++++++++++-------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 029b351ab37..569af4f5edd 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -121,16 +121,16 @@ class MockIMDClient: class IMDReader(StreamReaderBase): """ - Reader that supports the Interactive Molecular Dynamics (IMD) protocol for reading simulation - data using the IMDClient. + Reader that supports the Interactive Molecular Dynamics (IMD) protocol v3 for reading + simulation data using the IMDClient. - By using the keyword `buffer_size`, you can change the amount of memory the :class:`IMDClient` - allocates to its internal buffer. The buffer size determines how many frames can be stored - in memory as data is received from the socket and awaits reading by the client. For analyses - that periodically perform heavier computation at fixed intervals, say for example once every - 200 received frames, increasing this value will decrease the amount of time the simulation - engine spends in a paused state and potentially decrease total analysis time, but will require - more RAM. + By using the keyword `buffer_size`, you can change the amount of memory the + :class:`~imdclient.IMDClient.IMDClient` allocates to its internal buffer. + The buffer size determines how many frames can be stored in memory as data is received + from the socket and awaits reading by the client. For analyses that periodically perform + heavier computation at fixed intervals, say for example once every 200 received frames, + increasing this value will decrease the amount of time the simulation engine spends in a + paused state and potentially decrease total analysis time, but will require more RAM. Parameters ---------- @@ -141,10 +141,26 @@ class IMDReader(StreamReaderBase): number of atoms in the system. defaults to number of atoms in the topology. Don't set this unless you know what you're doing. buffer_size: int (optional) default=10*(1024**2) - number of bytes of memory to allocate to the :class:`IMDClient`'s + number of bytes of memory to allocate to the :class:`~imdclient.IMDClient.IMDClient`'s internal buffer. Defaults to 10 megabytes. kwargs : dict (optional) - keyword arguments passed to the constructed :class:`IMDClient` + keyword arguments passed to the constructed :class:`~imdclient.IMDClient.IMDClient` + + Notes + ----- + The IMDReader provides access to additional simulation data through the timestep's + `data` attribute (`ts.data`). The following keys may be available depending on + what the simulation engine transmits: + + * `dt` : float + Time step size in picoseconds (`IMD_TIME`_ of IMDv3 protocol) + * `step` : int + Current simulation step number (`IMD_TIME`_ of IMDv3 protocol) + * Energy terms : float + Various energy components (e.g., 'potential', 'kinetic', 'total', etc.) + (`IMD_ENERGIES` of the IMDv3 protocol). + + .. versionadded:: 2.10.0 """ format = "IMD" From 0ba7d0c00b81a6bc0dd87134984ecc01e6b91cba Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 4 Aug 2025 22:52:40 -0700 Subject: [PATCH 074/114] Remove: comment --- package/MDAnalysis/coordinates/IMD.py | 1 - 1 file changed, 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 569af4f5edd..0e7d98aea10 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -268,5 +268,4 @@ def close(self): logger.debug("IMDReader close() called") if self._imdclient is not None: self._imdclient.stop() - # NOTE: removeme after testing logger.debug("IMDReader shut down gracefully.") From 2e149aa0e84417f9903c4bef0d944b9b22c2dcdd Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 07:47:32 -0700 Subject: [PATCH 075/114] IMDReader docs: Links to protocol --- package/MDAnalysis/coordinates/IMD.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 0e7d98aea10..26e0e72bf0d 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -153,13 +153,17 @@ class IMDReader(StreamReaderBase): what the simulation engine transmits: * `dt` : float - Time step size in picoseconds (`IMD_TIME`_ of IMDv3 protocol) + Time step size in picoseconds (from the `IMD_TIME`_ packet of the IMDv3 protocol) * `step` : int - Current simulation step number (`IMD_TIME`_ of IMDv3 protocol) + Current simulation step number (from the `IMD_TIME`_ packet of the IMDv3 prtotocol) * Energy terms : float Various energy components (e.g., 'potential', 'kinetic', 'total', etc.) - (`IMD_ENERGIES` of the IMDv3 protocol). + from the `IMD_ENERGIES`_ packet of the IMDv3 protocol. + .. _IMD_TIME: https://imdclient.readthedocs.io/en/latest/protocol_v3.html#time + .. _IMD_ENERGIES: https://imdclient.readthedocs.io/en/latest/protocol_v3.html#energies + + .. versionadded:: 2.10.0 """ From 4aec1fe9665551baf4dcfd1764dcdb465840c2b1 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 07:52:00 -0700 Subject: [PATCH 076/114] Units removed --- package/MDAnalysis/coordinates/IMD.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 26e0e72bf0d..7f9f40cf15f 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -41,26 +41,7 @@ `imdclient `_ repository. -Units ------ -The units in IMDv3 are fixed. - -.. list-table:: - :widths: 10 10 - :header-rows: 1 - - * - Measurement - - Unit - * - Length - - angstrom - * - Velocity - - angstrom/picosecond - * - Force - - kilojoules/(mol*angstrom) - * - Time - - picosecond - * - Energy - - kilojoules/mol + Classes ------- From a481b56e442ed856a11fc8cb2c62ccaea2b9e7be Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 07:53:37 -0700 Subject: [PATCH 077/114] Chore: small chnage to IMDReader module docs --- package/MDAnalysis/coordinates/IMD.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 7f9f40cf15f..6e5189bc88b 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -25,7 +25,7 @@ gmx grompp -f run-NPT_imd-v3.mdp -c conf.gro -p topol.top -o topol.tpr gmx mdrun -v -nt 4 -imdwait -imdport 8889 -The :class:`MDAnalysis.coordinates.IMD.IMDReader` can then connect to the running simulation and stream data in real time: +The :class:`~MDAnalysis.coordinates.IMD.IMDReader` can then connect to the running simulation and stream data in real time: .. code-block:: python From d50d30725e6123ff17be11d433544bfb2be51b1b Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 07:55:49 -0700 Subject: [PATCH 078/114] Update: sphinx markup - `base.py` --- package/MDAnalysis/coordinates/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index ca4536a86a6..1e1c2079677 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1860,7 +1860,8 @@ class StreamReaderBase(ReaderBase): - **Forward-only**: Can only iterate sequentially through frames - **No copying**: Cannot create independent copies of the reader - The reader raises ``RuntimeError`` for operations that require random + + The reader raises :exc:`RuntimeError` for operations that require random access or rewinding, including ``rewind()``, ``copy()``, ``timeseries()``, and ``len()``. Only slice notation is supported for iteration. From 6a739ea9920ed0f3ce8c5935a7af019756670d3a Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 07:59:46 -0700 Subject: [PATCH 079/114] Added: See also doc --- package/MDAnalysis/coordinates/IMD.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 6e5189bc88b..cdcdf8a85d7 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -37,8 +37,12 @@ for ts in u.trajectory: print(f'{ts.time:8.3f} {sel[0].position} {sel[0].velocity} {sel[0].force} {u.dimensions[0:3]}') -Details about the IMD protocol and usage examples can be found in the -`imdclient `_ repository. +.. seealso:: + `imdclient documentation`_ and `github.com/Becksteinlab/imdclient`_ source code repository + +.. _`imdclient documentation`: https://imdclient.readthedocs.io/ +.. _`github.com/Becksteinlab/imdclient`: https://github.com/Becksteinlab/imdclient + From 4818a30c09316852374d36e798035e8f1d16eda6 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 08:03:03 -0700 Subject: [PATCH 080/114] Version Added: `StreamFrameIteratorSliced` --- package/MDAnalysis/coordinates/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 1e1c2079677..1a7f15784f6 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -2091,6 +2091,9 @@ class StreamFrameIteratorSliced(FrameIteratorBase): -------- StreamReaderBase FrameIteratorBase + + + .. versionadded:: 2.10.0 """ def __init__(self, trajectory, step): From a66122afc673e1b4387c1a13f1affe49c0a63942 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 08:06:34 -0700 Subject: [PATCH 081/114] seealso to `StreamReaderBase` added in `IMDReader` --- package/MDAnalysis/coordinates/IMD.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index cdcdf8a85d7..cd65c539cde 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -147,8 +147,12 @@ class IMDReader(StreamReaderBase): .. _IMD_TIME: https://imdclient.readthedocs.io/en/latest/protocol_v3.html#time .. _IMD_ENERGIES: https://imdclient.readthedocs.io/en/latest/protocol_v3.html#energies + + .. seealso:: + The IMDReader has some important limitations that are inherent in streaming data. + See :class:`~MDAnalysis.coordinates.base.StreamReaderBase` for details. - + .. versionadded:: 2.10.0 """ From 6ffbe378da289544b35166b5551ee05e76d5daeb Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 09:12:14 -0700 Subject: [PATCH 082/114] Update: module and class level docs for limitations --- package/MDAnalysis/coordinates/IMD.py | 82 +++++++++++++++++---------- 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index cd65c539cde..248eef3863f 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -2,19 +2,25 @@ IMDReader --- :mod:`MDAnalysis.coordinates.IMD` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:class:`MDAnalysis.coordinates.IMD.IMDReader` is a class that implements the -`Interactive Molecular Dynamics (IMD) protocol `_ for reading simulation -data using the IMDClient (see `imdclient `_). -The protocol allows two-way communicating molecular simulation data through a socket. +This module provides support for reading molecular dynamics simulation data via the +`Interactive Molecular Dynamics (IMD) protocol v3 `_. +The IMD protocol allows two-way communicating molecular simulation data through a socket. Via IMD, a simulation engine sends data to a receiver (in this case, the IMDClient) and the receiver can send forces and specific control requests (such as pausing, resuming, or terminating the simulation) back to the simulation engine. -IMDv3, the newest version of the protocol, is the one supported by this reader class and is implemented in GROMACS, LAMMPS, and NAMD at varying -stages of development. See the `imdclient simulation engine docs `_ for more. +.. note:: + This reader only supports IMDv3, which is implemented in GROMACS, LAMMPS, and NAMD at varying + stages of development. See the `imdclient simulation engine docs `_ for more. + While IMDv2 is widely available in simulation engines, it was designed primarily for visualization + and gaps are allowed in the stream (i.e., an inconsistent number of integrator time steps between transmitted coordinate arrays is allowed) -IMDv2, the first version to be broadly adopted, is currently available as a part of official releases of GROMACS, LAMMPS, and NAMD. However, -this reader class does not currently provide support for it since it was designed for visualization and gaps are allowed in the stream -(i.e., an inconsistent number of integrator time steps between transmitted coordinate arrays is allowed) +The :class:`IMDReader` connects to a simulation via a socket and receives coordinate, +velocity, force, and energy data as the simulation progresses. This allows for real-time +monitoring and analysis of ongoing simulations. It uses the `imdclient package `_ +(dependency) to implement the IMDv3 protocol and manage the socket connection and data parsing. + +Usage Example +------------- As an example of reading a stream, after configuring GROMACS to run a simulation with IMDv3 enabled (see the `imdclient simulation engine docs `_ for @@ -37,14 +43,30 @@ for ts in u.trajectory: print(f'{ts.time:8.3f} {sel[0].position} {sel[0].velocity} {sel[0].force} {u.dimensions[0:3]}') -.. seealso:: - `imdclient documentation`_ and `github.com/Becksteinlab/imdclient`_ source code repository +Important Limitations +--------------------- -.. _`imdclient documentation`: https://imdclient.readthedocs.io/ -.. _`github.com/Becksteinlab/imdclient`: https://github.com/Becksteinlab/imdclient +Since IMD streams data in real-time from a running simulation, there are some key +limitations to be aware of: +* **Forward-only access**: You can only move forward through frames as they arrive +* **No random access**: Cannot jump to arbitrary frame numbers or seek backwards +* **No trajectory length**: The total number of frames is unknown until the simulation ends +* **Timing dependent**: Analysis must keep up with the simulation's data rate +.. warning:: + The IMDReader has some important limitations that are inherent in streaming data. + See :class:`~MDAnalysis.coordinates.base.StreamReaderBase` for technical details. +.. seealso:: + :class:`IMDReader` + Technical details and parameter options for the reader class + + `imdclient documentation `_ + Complete documentation for the IMDClient package + + `IMDClient GitHub repository `_ + Source code and development resources Classes @@ -106,16 +128,16 @@ class MockIMDClient: class IMDReader(StreamReaderBase): """ - Reader that supports the Interactive Molecular Dynamics (IMD) protocol v3 for reading - simulation data using the IMDClient. + Coordinate reader implementing the IMDv3 protocol for streaming simulation data. + + This class handles the technical aspects of connecting to IMD-enabled simulation + engines and processing the incoming data stream. For usage examples and protocol + overview, see the module documentation above. - By using the keyword `buffer_size`, you can change the amount of memory the - :class:`~imdclient.IMDClient.IMDClient` allocates to its internal buffer. - The buffer size determines how many frames can be stored in memory as data is received - from the socket and awaits reading by the client. For analyses that periodically perform - heavier computation at fixed intervals, say for example once every 200 received frames, - increasing this value will decrease the amount of time the simulation engine spends in a - paused state and potentially decrease total analysis time, but will require more RAM. + The reader manages socket connections, data buffering, and frame parsing according + to the IMDv3 specification. It automatically handles different data packet types + (coordinates, velocities, forces, energies, timing) and populates MDAnalysis + timestep objects accordingly. Parameters ---------- @@ -127,8 +149,9 @@ class IMDReader(StreamReaderBase): in the topology. Don't set this unless you know what you're doing. buffer_size: int (optional) default=10*(1024**2) number of bytes of memory to allocate to the :class:`~imdclient.IMDClient.IMDClient`'s - internal buffer. Defaults to 10 megabytes. - kwargs : dict (optional) + internal buffer. Defaults to 10 megabytes. Larger buffers can improve + performance for analyses with periodic heavy computation. + **kwargs : dict (optional) keyword arguments passed to the constructed :class:`~imdclient.IMDClient.IMDClient` Notes @@ -140,7 +163,7 @@ class IMDReader(StreamReaderBase): * `dt` : float Time step size in picoseconds (from the `IMD_TIME`_ packet of the IMDv3 protocol) * `step` : int - Current simulation step number (from the `IMD_TIME`_ packet of the IMDv3 prtotocol) + Current simulation step number (from the `IMD_TIME`_ packet of the IMDv3 protocol) * Energy terms : float Various energy components (e.g., 'potential', 'kinetic', 'total', etc.) from the `IMD_ENERGIES`_ packet of the IMDv3 protocol. @@ -148,11 +171,10 @@ class IMDReader(StreamReaderBase): .. _IMD_TIME: https://imdclient.readthedocs.io/en/latest/protocol_v3.html#time .. _IMD_ENERGIES: https://imdclient.readthedocs.io/en/latest/protocol_v3.html#energies - .. seealso:: - The IMDReader has some important limitations that are inherent in streaming data. - See :class:`~MDAnalysis.coordinates.base.StreamReaderBase` for details. - - + .. note:: + For important limitations inherent to streaming data, see the module documentation above + and :class:`~MDAnalysis.coordinates.base.StreamReaderBase` for more technical details. + .. versionadded:: 2.10.0 """ From b404b047e74e1ec5665470d67a683fd9dd025aed Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 09:15:58 -0700 Subject: [PATCH 083/114] Remove `grompp` --- package/MDAnalysis/coordinates/IMD.py | 1 - 1 file changed, 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 248eef3863f..0698e24ff98 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -28,7 +28,6 @@ .. code-block:: bash - gmx grompp -f run-NPT_imd-v3.mdp -c conf.gro -p topol.top -o topol.tpr gmx mdrun -v -nt 4 -imdwait -imdport 8889 The :class:`~MDAnalysis.coordinates.IMD.IMDReader` can then connect to the running simulation and stream data in real time: From 0583c2eccba2e622ad2b68e24efe2c31f2c09e98 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 09:18:07 -0700 Subject: [PATCH 084/114] fix inlink link to imclient --- package/MDAnalysis/coordinates/IMD.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 0698e24ff98..45d33368253 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -10,7 +10,7 @@ .. note:: This reader only supports IMDv3, which is implemented in GROMACS, LAMMPS, and NAMD at varying - stages of development. See the `imdclient simulation engine docs `_ for more. + stages of development. See the `imdclient simulation engine docs`_ for more. While IMDv2 is widely available in simulation engines, it was designed primarily for visualization and gaps are allowed in the stream (i.e., an inconsistent number of integrator time steps between transmitted coordinate arrays is allowed) @@ -23,7 +23,7 @@ ------------- As an example of reading a stream, after configuring GROMACS to run a simulation with IMDv3 enabled -(see the `imdclient simulation engine docs `_ for +(see the `imdclient simulation engine docs`_ for up-to-date resources on configuring each simulation engine), use the following commands: .. code-block:: bash @@ -67,6 +67,8 @@ `IMDClient GitHub repository `_ Source code and development resources +.. _`imdclient simulation engine docs`: https://imdclient.readthedocs.io/en/latest/usage.html + Classes ------- From 73d78521e31af54bc1aa9415a21186685f59e4b5 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 09:21:16 -0700 Subject: [PATCH 085/114] Update CHANGELOG w `StreamReaderBase` --- package/CHANGELOG | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package/CHANGELOG b/package/CHANGELOG index 6a51da10b80..ca3ad793226 100644 --- a/package/CHANGELOG +++ b/package/CHANGELOG @@ -63,9 +63,11 @@ Enhancements (PR #5038) * Moved distopia checking function to common import location in MDAnalysisTest.util (PR #5038) + * Added support for reading and processing streamed data in `coordinates.base` + with new `StreamFrameIteratorSliced` and `StreamReaderBase` (Issue #4827, PR #4923) * New coordinate reader: Added `IMDReader` for reading real-time streamed molecular dynamics simulation data using the IMDv3 protocol - requires - `imdclient` package (Issue #4827, PR #4923) + `imdclient` package (Issue #4827, PR #4923) Changes * Refactored the RDKit converter code to move the inferring code in a separate From f04c43441767b9ab56a4efadb308803ef3f07661 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Wed, 6 Aug 2025 09:23:45 -0700 Subject: [PATCH 086/114] Minor clean up --- package/MDAnalysis/coordinates/IMD.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 45d33368253..365b5fe523c 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -236,10 +236,7 @@ def __init__( def _read_frame(self, frame): - try: - imdf = self._imdclient.get_imdframe() - except EOFError as e: - raise e + imdf = self._imdclient.get_imdframe() self._frame = frame self._load_imdframe_into_ts(imdf) From 474981daef1c4a3e6785e92ab116317cd4a62ca9 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 7 Aug 2025 11:43:16 -0700 Subject: [PATCH 087/114] Typo: precision --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 8e63630192f..2aef52f96cb 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -186,7 +186,7 @@ def test_volume(self, ref, reader): # don't rewind here as in inherited base test vol = reader.ts.volume # Here we can only be sure about the numbers upto the decimal point due - # to floating point impressions. + # to limited floating point precision. assert_allclose(vol, ref.volume, rtol=0, atol=1.5e0) def test_reload_auxiliaries_from_description(self, ref, reader): From e32df0d9dc37f9b0ac5623e741c46b6b2793c579 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 7 Aug 2025 11:43:42 -0700 Subject: [PATCH 088/114] Limitations updated: `IMDReader` and `StreamReaderBase` --- package/MDAnalysis/coordinates/IMD.py | 14 ++++++++++---- package/MDAnalysis/coordinates/base.py | 8 ++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 365b5fe523c..fcd2a98b60c 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -45,13 +45,19 @@ Important Limitations --------------------- -Since IMD streams data in real-time from a running simulation, there are some key -limitations to be aware of: +Since IMD streams data in real-time from a running simulation, it has fundamental +constraints that differ from traditional trajectory readers: -* **Forward-only access**: You can only move forward through frames as they arrive * **No random access**: Cannot jump to arbitrary frame numbers or seek backwards +* **Forward-only access**: You can only move forward through frames as they arrive * **No trajectory length**: The total number of frames is unknown until the simulation ends -* **Timing dependent**: Analysis must keep up with the simulation's data rate +* **Single-use iteration**: Cannot restart or rewind once the stream has been consumed +* **No independent copies**: Cannot create separate reader instances for the same stream +* **No stream restart**: Cannot reconnect or reopen once the connection is closed +* **No bulk operations**: Cannot extract all data at once using timeseries methods +* **Limited multiprocessing**: Cannot split reader across processes for parallel analysis +* **Single client connection**: Only one reader can connect to an IMD stream at a time +* **No trajectory Writing**: Complimentary IMD Writer class is not available for streaming data .. warning:: The IMDReader has some important limitations that are inherent in streaming data. diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 1a7f15784f6..612ad4bc76e 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1855,10 +1855,14 @@ class StreamReaderBase(ReaderBase): that can randomly access frames, streaming readers have fundamental constraints: - **No random access**: Cannot seek to arbitrary frames (no ``traj[5]``) - - **No rewinding**: Cannot restart or rewind the stream - - **No length**: Total number of frames is unknown until stream ends - **Forward-only**: Can only iterate sequentially through frames + - **No length**: Total number of frames is unknown until stream ends + - **No rewinding**: Cannot restart or rewind the stream - **No copying**: Cannot create independent copies of the reader + - **No reopening**: Cannot restart iteration once stream is consumed + - **No timeseries**: Cannot use ``timeseries()`` or bulk data extraction + - **No pickling**: Cannot serialize reader instances (limits multiprocessing) + - **No WriterBase**: No complementary Writer class available for streaming data The reader raises :exc:`RuntimeError` for operations that require random From e4211a1187a8a28791de83e714b7d9026b32a7ce Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 7 Aug 2025 12:01:28 -0700 Subject: [PATCH 089/114] API Doc: `check_slice_indices`` --- package/MDAnalysis/coordinates/base.py | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 612ad4bc76e..0b9008b719b 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -2014,6 +2014,62 @@ def __getitem__(self, frame): ) def check_slice_indices(self, start, stop, step): + """Check and validate slice indices for streaming trajectories. + + Streaming trajectories have fundamental constraints that differ from + traditional trajectory files: + + * **No start/stop indices**: Since streams process data continuously + without knowing the total length, ``start`` and ``stop`` must be ``None`` + * **Step-only slicing**: Only the ``step`` parameter is meaningful, + controlling how many frames to skip during iteration + * **Forward-only**: ``step`` must be positive (> 0) as streams cannot + be processed backward in time + + Parameters + ---------- + start : int or None + Starting frame index. Must be ``None`` for streaming readers. + stop : int or None + Ending frame index. Must be ``None`` for streaming readers. + step : int or None + Step size for iteration. Must be positive integer or ``None`` + (equivalent to 1). + + Returns + ------- + tuple + (start, stop, step) with validated values + + Raises + ------ + ValueError + If ``start`` or ``stop`` are not ``None``, or if ``step`` is + not a positive integer. + + Examples + -------- + Valid streaming slices:: + + traj[:] # All frames (step=None, equivalent to step=1) + traj[::2] # Every 2nd frame + traj[::10] # Every 10th frame + + Invalid streaming slices:: + + traj[5:] # Cannot specify start index + traj[:100] # Cannot specify stop index + traj[5:100:2] # Cannot specify start or stop indices + traj[::-1] # Cannot go backwards (negative step) + + See Also + -------- + __getitem__ + StreamFrameIteratorSliced + + + .. versionadded:: 2.10.0 + """ if start is not None: raise ValueError( "{}: Cannot expect a start index from a stream, 'start' must be None".format( From 7aa10d676dc4c49a42d872144d2e3e5c52edbd99 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 7 Aug 2025 12:07:16 -0700 Subject: [PATCH 090/114] API Doc: `timeseries` --- package/MDAnalysis/coordinates/base.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 0b9008b719b..ce23e58bad2 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1965,6 +1965,23 @@ def _reopen(self): self._reopen_called = True def timeseries(self, **kwargs): + """Timeseries extraction is not supported for streaming trajectories. + + Streaming readers cannot randomly access frames or store bulk coordinate + data in memory, which ``timeseries()`` requires. Use sequential frame + iteration instead. + + Parameters + ---------- + **kwargs + Any keyword arguments (ignored, as method is not supported) + + Raises + ------ + RuntimeError + Always raised, as timeseries extraction is not supported for + streaming trajectories + """ raise RuntimeError( "{}: cannot access timeseries for streamed trajectories".format(self.__class__.__name__) ) From ab3854833674e99f91456d3a08d4944bf2425642 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 7 Aug 2025 12:09:49 -0700 Subject: [PATCH 091/114] API Doc: `copy` --- package/MDAnalysis/coordinates/base.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index ce23e58bad2..8166e452421 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1952,6 +1952,17 @@ def rewind(self): # Incompatible methods def copy(self): + """Reader copying is not supported for streaming trajectories. + + Streaming readers maintain internal state and connection resources + that cannot be duplicated. Each stream connection is unique and + cannot be copied. + + Raises + ------ + RuntimeError + Always raised, as copying is not supported for streaming trajectories + """ raise RuntimeError( "{} does not support copying".format(self.__class__.__name__) ) From b63df7e45103086946e357c3d0e3e640b7413335 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 7 Aug 2025 12:12:06 -0700 Subject: [PATCH 092/114] API Doc: `rewind` --- package/MDAnalysis/coordinates/base.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index 8166e452421..a890f2d2805 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1943,7 +1943,16 @@ def next(self): return ts def rewind(self): - """Raise error on rewind""" + """Rewinding is not supported for streaming trajectories. + + Streaming readers process data continuously from streams + and cannot restart or go backward in the stream once consumed. + + Raises + ------ + RuntimeError + Always raised, as rewinding is not supported for streaming trajectories + """ raise RuntimeError( "{}: Stream-based readers can't be rewound".format( self.__class__.__name__ From d663d8953997b19b00ddebc9d8a0c8488e84a98f Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 7 Aug 2025 12:25:46 -0700 Subject: [PATCH 093/114] API Doc: `next` --- package/MDAnalysis/coordinates/base.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index a890f2d2805..d84fef0d2b9 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1926,8 +1926,20 @@ def __len__(self): ) def next(self): - """Don't rewind after iteration. When _reopen() is called, - an error will be raised + """Advance to the next timestep in the streaming trajectory. + + Streaming readers process frames sequentially and cannot rewind + once iteration completes. Use ``for ts in trajectory`` for iteration. + + Returns + ------- + Timestep + The next timestep in the stream + + Raises + ------ + StopIteration + When the stream ends or no more frames are available """ try: ts = self._read_next_timestep() From 87311d013f81fa5b0d5fd84b563302566072284d Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 7 Aug 2025 12:25:58 -0700 Subject: [PATCH 094/114] API Doc: `_reopen` --- package/MDAnalysis/coordinates/base.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index d84fef0d2b9..e0a7db45e1a 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1989,6 +1989,17 @@ def copy(self): ) def _reopen(self): + """Prepare stream for iteration - can only be called once. + + Streaming readers cannot be reopened once iteration begins. + This method is called internally during iteration setup and + will raise an error if called multiple times. + + Raises + ------ + RuntimeError + If the stream has already been opened for iteration + """ if self._reopen_called: raise RuntimeError( "{}: Cannot reopen stream".format(self.__class__.__name__) From 6aee9154d2b8c2e30426bb4cf288b4b5ae6ca118 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 7 Aug 2025 12:51:19 -0700 Subject: [PATCH 095/114] black `IMD.py` --- package/MDAnalysis/coordinates/IMD.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index fcd2a98b60c..3cd46860c8b 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -136,7 +136,7 @@ class MockIMDClient: class IMDReader(StreamReaderBase): """ Coordinate reader implementing the IMDv3 protocol for streaming simulation data. - + This class handles the technical aspects of connecting to IMD-enabled simulation engines and processing the incoming data stream. For usage examples and protocol overview, see the module documentation above. From 350af5fd8695a7442e9e89db3aecdeb5b266b85f Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 7 Aug 2025 13:10:12 -0700 Subject: [PATCH 096/114] black `test_imd.py` --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 2aef52f96cb..805851573c5 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -39,6 +39,7 @@ COORDINATES_TRR, ) + @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class IMDReference(BaseReference): def __init__(self): From 0deecc09a9b8c5e65fe0850cff596441e7f5c3cd Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Tue, 12 Aug 2025 20:59:05 -0700 Subject: [PATCH 097/114] Error raised and tested in `test_imd.py` --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 805851573c5..753459d51a3 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -141,7 +141,8 @@ def transformed(ref): return transformed def test_n_frames(self, ref, reader): - pytest.skip("`n_frames` is unknown for IMDReader") + with pytest.raises(RuntimeError, match="n_frames is unknown"): + reader.n_frames def test_first_frame(self, ref, reader): # don't rewind here as in inherited base test @@ -159,7 +160,8 @@ def test_total_time(self, ref, reader): pytest.skip("`total_time` is unknown for IMDReader") def test_changing_dimensions(self, ref, reader): - pytest.skip("IMDReader cannot be rewound") + with pytest.raises(RuntimeError, match="Stream-based readers can't be rewound"): + reader.rewind() def test_iter(self, ref, reader): for i, ts in enumerate(reader): @@ -194,7 +196,8 @@ def test_reload_auxiliaries_from_description(self, ref, reader): pytest.skip("Cannot create two IMDReaders on the same stream") def test_stop_iter(self, reader): - pytest.skip("IMDReader cannot be rewound") + with pytest.raises(RuntimeError, match="Stream-based readers can't be rewound"): + reader.rewind() def test_iter_rewinds(self, reader): pytest.skip("IMDReader cannot be rewound") @@ -221,7 +224,8 @@ def test_transformation_rewind(self, ref, transformed): pytest.skip("IMDReader cannot be reopened") def test_pickle_reader(self, reader): - pytest.skip("IMDReader cannot be pickled") + with pytest.raises(NotImplementedError, match="does not support pickling"): + pickle.dumps(reader) def test_pickle_next_ts_reader(self, reader): pytest.skip("IMDReader cannot be pickled") @@ -230,7 +234,8 @@ def test_pickle_last_ts_reader(self, reader): pytest.skip("IMDReader cannot be pickled") def test_transformations_copy(self, ref, transformed): - pytest.skip("IMDReader cannot be copied") + with pytest.raises(RuntimeError, match="does not support copying"): + transformed.copy() def test_timeseries_empty_asel(self, reader): pytest.skip("IMDReader does not support timeseries") From 54f17fd9123731ff83b2b081423fd8f15a072cf9 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Tue, 12 Aug 2025 21:50:38 -0700 Subject: [PATCH 098/114] `step`: docstring and test coverage --- package/MDAnalysis/coordinates/base.py | 12 ++++++++++++ testsuite/MDAnalysisTests/coordinates/test_imd.py | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index e0a7db45e1a..ecb6047f854 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -2248,4 +2248,16 @@ def __getitem__(self, frame): @property def step(self): + """The step size for sliced frame iteration. + + Returns the step interval used when iterating through frames in a + streaming trajectory. For example, a step of 2 means every second + frame is processed, while a step of 1 processes every frame. + + Returns + ------- + int + Step size for iteration. Always a positive integer greater than 0. + + """ return self._step \ No newline at end of file diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 753459d51a3..893dd7324c3 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -363,6 +363,15 @@ def test_timeseries_raises(self, reader): ): reader.timeseries() + def test_step_property(self, reader): + """Test that the step property returns the correct step size.""" + # Test step property for different slice steps + sliced_reader = reader[::1] + assert sliced_reader.step == 1 + + sliced_reader_step5 = reader[::5] + assert sliced_reader_step5.step == 5 + @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") def test_n_atoms_not_specified(universe, imdsinfo): From 849b8db60bcf2f1f4781ea285db5888877836065 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Tue, 12 Aug 2025 21:51:08 -0700 Subject: [PATCH 099/114] `StreamReaderBase` writer doctsring and tests --- package/MDAnalysis/coordinates/base.py | 54 ++++++++++++++++++- .../MDAnalysisTests/coordinates/test_imd.py | 6 ++- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/package/MDAnalysis/coordinates/base.py b/package/MDAnalysis/coordinates/base.py index ecb6047f854..d2bfbe63d2c 100644 --- a/package/MDAnalysis/coordinates/base.py +++ b/package/MDAnalysis/coordinates/base.py @@ -1861,13 +1861,14 @@ class StreamReaderBase(ReaderBase): - **No copying**: Cannot create independent copies of the reader - **No reopening**: Cannot restart iteration once stream is consumed - **No timeseries**: Cannot use ``timeseries()`` or bulk data extraction + - **No writers**: Cannot create ``Writer()`` or ``OtherWriter()`` instances - **No pickling**: Cannot serialize reader instances (limits multiprocessing) - - **No WriterBase**: No complementary Writer class available for streaming data + - **No StreamWriterBase**: No complementary Writer class available for streaming data The reader raises :exc:`RuntimeError` for operations that require random access or rewinding, including ``rewind()``, ``copy()``, ``timeseries()``, - and ``len()``. Only slice notation is supported for iteration. + ``Writer()``, ``OtherWriter()``, and ``len()``. Only slice notation is supported for iteration. Parameters ---------- @@ -2159,6 +2160,55 @@ def check_slice_indices(self, start, stop, step): return start, stop, step + def Writer(self, filename, **kwargs): + """Writer creation is not supported for streaming trajectories. + + Writer creation requires trajectory metadata that streaming readers + cannot provide due to their sequential processing nature. + + Parameters + ---------- + filename : str + Output filename (ignored, as method is not supported) + **kwargs + Additional keyword arguments (ignored, as method is not supported) + + Raises + ------ + RuntimeError + Always raised, as writer creation is not supported for streaming trajectories + """ + raise RuntimeError( + "{}: cannot create Writer for streamed trajectories".format( + self.__class__.__name__ + ) + ) + + def OtherWriter(self, filename, **kwargs): + """Writer creation is not supported for streaming trajectories. + + OtherWriter initialization requires frame-based parameters and trajectory + indexing information. Streaming readers process data sequentially + without meaningful frame indexing, making writer setup impossible. + + Parameters + ---------- + filename : str + Output filename (ignored, as method is not supported) + **kwargs + Additional keyword arguments (ignored, as method is not supported) + + Raises + ------ + RuntimeError + Always raised, as writer creation is not supported for streaming trajectories + """ + raise RuntimeError( + "{}: cannot create OtherWriter for streamed trajectories".format( + self.__class__.__name__ + ) + ) + def __getstate__(self): raise NotImplementedError( "{} does not support pickling".format(self.__class__.__name__) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 893dd7324c3..85e04cfb7cb 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -151,10 +151,12 @@ def test_first_frame(self, ref, reader): ) def test_get_writer_1(self, ref, reader, tmpdir): - pytest.skip("No Writer for IMDReader") + with pytest.raises(RuntimeError, match="cannot create Writer for streamed trajectories"): + reader.Writer(str(tmpdir.join("output"))) def test_get_writer_2(self, ref, reader, tmpdir): - pytest.skip("No Writer for IMDReader") + with pytest.raises(RuntimeError, match="cannot create Writer for streamed trajectories"): + reader.Writer(str(tmpdir.join("output")), n_atoms=100) def test_total_time(self, ref, reader): pytest.skip("`total_time` is unknown for IMDReader") From 487753e2543754ea9d3472cc481d04094a46606f Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Tue, 12 Aug 2025 22:12:27 -0700 Subject: [PATCH 100/114] `test_wrong_imd_protocol_version` --- .../MDAnalysisTests/coordinates/test_imd.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 85e04cfb7cb..5d6eb743458 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -432,3 +432,23 @@ def test_imd_format_hint(): assert not IMDReader._format_hint("not_a_valid_imd_url") assert not IMDReader._format_hint(12345) assert not IMDReader._format_hint(None) + + +@pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") +def test_wrong_imd_protocol_version(universe, imdsinfo): + """Test that IMDReader raises ValueError for non-v3 protocol versions.""" + # Modify the fixture to have wrong version + imdsinfo.version = 2 # Wrong version, should be 3 + + server = InThreadIMDServer(universe.trajectory) + server.set_imdsessioninfo(imdsinfo) + server.handshake_sequence("localhost", first_frame=True) + + with pytest.raises(ValueError, + match=rf"IMDReader: Detected IMD version v{imdsinfo.version}, " + rf"but IMDReader is only compatible with v3"): + IMDReader( + f"imd://localhost:{server.port}", + n_atoms=universe.trajectory.n_atoms, + ) + server.cleanup() From a9d4fd4b6f1579168c50318a7809b3044ae7bbf8 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Tue, 12 Aug 2025 22:14:11 -0700 Subject: [PATCH 101/114] black `test_imd.py` --- .../MDAnalysisTests/coordinates/test_imd.py | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 5d6eb743458..0b7c17d49dc 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -151,18 +151,26 @@ def test_first_frame(self, ref, reader): ) def test_get_writer_1(self, ref, reader, tmpdir): - with pytest.raises(RuntimeError, match="cannot create Writer for streamed trajectories"): + with pytest.raises( + RuntimeError, + match="cannot create Writer for streamed trajectories", + ): reader.Writer(str(tmpdir.join("output"))) def test_get_writer_2(self, ref, reader, tmpdir): - with pytest.raises(RuntimeError, match="cannot create Writer for streamed trajectories"): + with pytest.raises( + RuntimeError, + match="cannot create Writer for streamed trajectories", + ): reader.Writer(str(tmpdir.join("output")), n_atoms=100) def test_total_time(self, ref, reader): pytest.skip("`total_time` is unknown for IMDReader") def test_changing_dimensions(self, ref, reader): - with pytest.raises(RuntimeError, match="Stream-based readers can't be rewound"): + with pytest.raises( + RuntimeError, match="Stream-based readers can't be rewound" + ): reader.rewind() def test_iter(self, ref, reader): @@ -198,7 +206,9 @@ def test_reload_auxiliaries_from_description(self, ref, reader): pytest.skip("Cannot create two IMDReaders on the same stream") def test_stop_iter(self, reader): - with pytest.raises(RuntimeError, match="Stream-based readers can't be rewound"): + with pytest.raises( + RuntimeError, match="Stream-based readers can't be rewound" + ): reader.rewind() def test_iter_rewinds(self, reader): @@ -226,7 +236,9 @@ def test_transformation_rewind(self, ref, transformed): pytest.skip("IMDReader cannot be reopened") def test_pickle_reader(self, reader): - with pytest.raises(NotImplementedError, match="does not support pickling"): + with pytest.raises( + NotImplementedError, match="does not support pickling" + ): pickle.dumps(reader) def test_pickle_next_ts_reader(self, reader): @@ -370,7 +382,7 @@ def test_step_property(self, reader): # Test step property for different slice steps sliced_reader = reader[::1] assert sliced_reader.step == 1 - + sliced_reader_step5 = reader[::5] assert sliced_reader_step5.step == 5 @@ -439,14 +451,16 @@ def test_wrong_imd_protocol_version(universe, imdsinfo): """Test that IMDReader raises ValueError for non-v3 protocol versions.""" # Modify the fixture to have wrong version imdsinfo.version = 2 # Wrong version, should be 3 - + server = InThreadIMDServer(universe.trajectory) server.set_imdsessioninfo(imdsinfo) server.handshake_sequence("localhost", first_frame=True) - - with pytest.raises(ValueError, - match=rf"IMDReader: Detected IMD version v{imdsinfo.version}, " - rf"but IMDReader is only compatible with v3"): + + with pytest.raises( + ValueError, + match=rf"IMDReader: Detected IMD version v{imdsinfo.version}, " + rf"but IMDReader is only compatible with v3", + ): IMDReader( f"imd://localhost:{server.port}", n_atoms=universe.trajectory.n_atoms, From aa395dc09c3df21e11d813309780b05e500adb07 Mon Sep 17 00:00:00 2001 From: Oliver Beckstein Date: Wed, 27 Aug 2025 11:01:02 -0700 Subject: [PATCH 102/114] added test for RuntimeError on IMDReader.OtherWriter --- testsuite/MDAnalysisTests/coordinates/test_imd.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 0b7c17d49dc..29e713fc639 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -164,6 +164,13 @@ def test_get_writer_2(self, ref, reader, tmpdir): ): reader.Writer(str(tmpdir.join("output")), n_atoms=100) + def test_OtherWriter_RuntimeError(self, reader, tmpdir): + with pytest.raises( + RuntimeError, + match="cannot create OtherWriter for streamed trajectories", + ): + reader.OtherWriter(tmpdir.join("output")) + def test_total_time(self, ref, reader): pytest.skip("`total_time` is unknown for IMDReader") From 66347068676ea7f3db4a82147625d428e785bd5f Mon Sep 17 00:00:00 2001 From: Oliver Beckstein Date: Wed, 27 Aug 2025 11:10:49 -0700 Subject: [PATCH 103/114] merged test_imd_import into test_imd --- .../MDAnalysisTests/coordinates/test_imd.py | 62 ++++++++++++++- .../coordinates/test_imd_import.py | 75 ------------------- 2 files changed, 61 insertions(+), 76 deletions(-) delete mode 100644 testsuite/MDAnalysisTests/coordinates/test_imd_import.py diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 29e713fc639..6e8d579422a 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -17,7 +17,11 @@ ) import MDAnalysis as mda -from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT, IMDReader +from MDAnalysis.coordinates.IMD import ( + HAS_IMDCLIENT, + MIN_IMDCLIENT_VERSION, + IMDReader, +) from MDAnalysis.transformations import translate if HAS_IMDCLIENT: @@ -40,6 +44,62 @@ ) +def test_IMDCLIENT_import(monkeypatch): + backup = sys.modules.copy() + + try: + module_name = "imdclient" + + # Create mock modules + mocked_module = ModuleType(module_name) + IMDClient_module = ModuleType(f"{module_name}.IMDClient") + + class MockIMDClient: + pass + + IMDClient_module.IMDClient = MockIMDClient + mocked_module.IMDClient = IMDClient_module + mocked_module.__version__ = str(MIN_IMDCLIENT_VERSION) + + utils_module = ModuleType(f"{module_name}.utils") + utils_module.parse_host_port = lambda x: ("localhost", 12345) + mocked_module.utils = utils_module + + monkeypatch.setitem(sys.modules, module_name, mocked_module) + monkeypatch.setitem( + sys.modules, f"{module_name}.IMDClient", IMDClient_module + ) + monkeypatch.setitem(sys.modules, f"{module_name}.utils", utils_module) + + sys.modules.pop("MDAnalysis.coordinates.IMD", None) + + # check if imdclient is new enough + import MDAnalysis.coordinates.IMD + + importlib.reload(MDAnalysis.coordinates.IMD) + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + + assert HAS_IMDCLIENT + + # check if imdclient version is too old + mocked_module.__version__ = "0.0.0" + importlib.reload(MDAnalysis.coordinates.IMD) + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + from MDAnalysis.coordinates.IMD import IMDReader as IMDReader_NOClient + + assert not HAS_IMDCLIENT + + # test initialization error + with pytest.raises( + ImportError, match="IMDReader requires the imdclient" + ): + IMDReader_NOClient("imd://localhost:12345", n_atoms=5) + finally: + # Restore sys.modules to avoid side effects on other tests + sys.modules.clear() + sys.modules.update(backup) + + @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") class IMDReference(BaseReference): def __init__(self): diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd_import.py b/testsuite/MDAnalysisTests/coordinates/test_imd_import.py deleted file mode 100644 index fe2698d0cdf..00000000000 --- a/testsuite/MDAnalysisTests/coordinates/test_imd_import.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Test for MDAnalysis trajectory reader expectations -""" - -import sys -import importlib -import pytest -from types import ModuleType - -from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT, MIN_IMDCLIENT_VERSION - -if HAS_IMDCLIENT: - import imdclient - from imdclient.tests.utils import ( - get_free_port, - create_default_imdsinfo_v3, - ) - from imdclient.tests.server import InThreadIMDServer - -from MDAnalysis.coordinates.IMD import IMDReader - - -def test_IMDCLIENT_import(monkeypatch): - backup = sys.modules.copy() - - try: - module_name = "imdclient" - - # Create mock modules - mocked_module = ModuleType(module_name) - IMDClient_module = ModuleType(f"{module_name}.IMDClient") - - class MockIMDClient: - pass - - IMDClient_module.IMDClient = MockIMDClient - mocked_module.IMDClient = IMDClient_module - mocked_module.__version__ = str(MIN_IMDCLIENT_VERSION) - - utils_module = ModuleType(f"{module_name}.utils") - utils_module.parse_host_port = lambda x: ("localhost", 12345) - mocked_module.utils = utils_module - - monkeypatch.setitem(sys.modules, module_name, mocked_module) - monkeypatch.setitem( - sys.modules, f"{module_name}.IMDClient", IMDClient_module - ) - monkeypatch.setitem(sys.modules, f"{module_name}.utils", utils_module) - - sys.modules.pop("MDAnalysis.coordinates.IMD", None) - - # check if imdclient is new enough - import MDAnalysis.coordinates.IMD - - importlib.reload(MDAnalysis.coordinates.IMD) - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - - assert HAS_IMDCLIENT - - # check if imdclient version is too old - mocked_module.__version__ = "0.0.0" - importlib.reload(MDAnalysis.coordinates.IMD) - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - from MDAnalysis.coordinates.IMD import IMDReader as IMDReader_NOClient - - assert not HAS_IMDCLIENT - - # test initialization error - with pytest.raises( - ImportError, match="IMDReader requires the imdclient" - ): - IMDReader_NOClient("imd://localhost:12345", n_atoms=5) - finally: - # Restore sys.modules to avoid side effects on other tests - sys.modules.clear() - sys.modules.update(backup) From 04424a8719a39937fd8147fd3678699965b070b6 Mon Sep 17 00:00:00 2001 From: Oliver Beckstein Date: Wed, 27 Aug 2025 12:36:54 -0700 Subject: [PATCH 104/114] refactored IMD import test to use a custom state manager - testing imports of IMD/imdclient module is now isolated from other tests - testing different versions of imdclient and IMD.HAS_IMDCLIENT is now done in separate tests --- .../MDAnalysisTests/coordinates/test_imd.py | 108 +++++++++++++----- 1 file changed, 81 insertions(+), 27 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 6e8d579422a..2fed2f761fd 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -44,13 +44,52 @@ ) -def test_IMDCLIENT_import(monkeypatch): - backup = sys.modules.copy() +class IMDModuleStateManager: + """Context manager to completely backup and restore imdclient/IMD module state. + + We need a custom manager because IMD changes its own state (HAS_IMDCLIENT) when it is imported + and we are going to manipulate the state of the imdclient module that IMD sees. + """ + + def __init__(self): + self.original_modules = None + self.imd_was_imported = False + + def __enter__(self): + # Backup sys.modules + self.original_modules = sys.modules.copy() + + # Check if IMD module was already imported + self.imd_was_imported = "MDAnalysis.coordinates.IMD" in sys.modules + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Restore sys.modules completely first + sys.modules.clear() + sys.modules.update(self.original_modules) + + # If IMD module was originally imported, force a fresh reload to restore original state + # This ensures that HAS_IMDCLIENT and other globals are recalculated with the real imdclient + if self.imd_was_imported: + # Remove the potentially corrupted IMD module + sys.modules.pop("MDAnalysis.coordinates.IMD", None) + # Fresh import will re-evaluate all globals + import MDAnalysis.coordinates.IMD - try: - module_name = "imdclient" - # Create mock modules +class TestImport: + """Test imdclient import behavior and HAS_IMDCLIENT flag.""" + + def _setup_mock_imdclient(self, monkeypatch, version): + """Helper method to set up mock imdclient with specified version.""" + # Remove IMD and imdclient modules to force fresh import + monkeypatch.delitem( + sys.modules, "MDAnalysis.coordinates.IMD", raising=False + ) + monkeypatch.delitem(sys.modules, "imdclient", raising=False) + + module_name = "imdclient" mocked_module = ModuleType(module_name) IMDClient_module = ModuleType(f"{module_name}.IMDClient") @@ -59,7 +98,7 @@ class MockIMDClient: IMDClient_module.IMDClient = MockIMDClient mocked_module.IMDClient = IMDClient_module - mocked_module.__version__ = str(MIN_IMDCLIENT_VERSION) + mocked_module.__version__ = version utils_module = ModuleType(f"{module_name}.utils") utils_module.parse_host_port = lambda x: ("localhost", 12345) @@ -71,33 +110,48 @@ class MockIMDClient: ) monkeypatch.setitem(sys.modules, f"{module_name}.utils", utils_module) - sys.modules.pop("MDAnalysis.coordinates.IMD", None) + return mocked_module - # check if imdclient is new enough - import MDAnalysis.coordinates.IMD + def test_has_minversion(self, monkeypatch): + """Test that HAS_IMDCLIENT is True when imdclient >= MIN_IMDCLIENT_VERSION.""" + with IMDModuleStateManager(): + self._setup_mock_imdclient(monkeypatch, str(MIN_IMDCLIENT_VERSION)) - importlib.reload(MDAnalysis.coordinates.IMD) - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + # Import and check HAS_IMDCLIENT with compatible version + import MDAnalysis.coordinates.IMD + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - assert HAS_IMDCLIENT + assert ( + HAS_IMDCLIENT + ), f"HAS_IMDCLIENT should be True with version {MIN_IMDCLIENT_VERSION}" - # check if imdclient version is too old - mocked_module.__version__ = "0.0.0" - importlib.reload(MDAnalysis.coordinates.IMD) - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - from MDAnalysis.coordinates.IMD import IMDReader as IMDReader_NOClient + def test_no_minversion(self, monkeypatch): + """Test that HAS_IMDCLIENT is False when imdclient version is too old.""" + with IMDModuleStateManager(): + self._setup_mock_imdclient(monkeypatch, "0.0.0") - assert not HAS_IMDCLIENT + # Import and check HAS_IMDCLIENT with incompatible version + import MDAnalysis.coordinates.IMD + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - # test initialization error - with pytest.raises( - ImportError, match="IMDReader requires the imdclient" - ): - IMDReader_NOClient("imd://localhost:12345", n_atoms=5) - finally: - # Restore sys.modules to avoid side effects on other tests - sys.modules.clear() - sys.modules.update(backup) + assert ( + not HAS_IMDCLIENT + ), "HAS_IMDCLIENT should be False with version 0.0.0" + + def test_missing_ImportError(self, monkeypatch): + """Test that IMDReader raises ImportError when HAS_IMDCLIENT=False.""" + with IMDModuleStateManager(): + self._setup_mock_imdclient(monkeypatch, "0.0.0") + + # Import with incompatible version (HAS_IMDCLIENT=False) + import MDAnalysis.coordinates.IMD + from MDAnalysis.coordinates.IMD import IMDReader + + # IMDReader should raise ImportError when HAS_IMDCLIENT=False + with pytest.raises( + ImportError, match="IMDReader requires the imdclient" + ): + IMDReader("imd://localhost:12345", n_atoms=5) @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") From d68de4270ce0ce6808cd7bd3e120541ff83787fa Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Wed, 27 Aug 2025 14:00:44 -0700 Subject: [PATCH 105/114] remove __test__ --- testsuite/MDAnalysisTests/coordinates/test_reader_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_reader_api.py b/testsuite/MDAnalysisTests/coordinates/test_reader_api.py index b5548f0ac1e..2e6e29c852a 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_reader_api.py +++ b/testsuite/MDAnalysisTests/coordinates/test_reader_api.py @@ -472,8 +472,6 @@ class _Stream: class TestStreamReader(_Stream): - __test__ = True - @pytest.fixture def reader(self): return self.readerclass("dummy", n_atoms=self.n_atoms) From 16454a624b6dacad197bbbc1382778a5ab88da5d Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Thu, 28 Aug 2025 10:51:46 -0700 Subject: [PATCH 106/114] Modify: `timestep.pyx` to remove usage of `new` keyword --- package/MDAnalysis/coordinates/timestep.pyx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package/MDAnalysis/coordinates/timestep.pyx b/package/MDAnalysis/coordinates/timestep.pyx index ee12feae375..09e44c0f551 100644 --- a/package/MDAnalysis/coordinates/timestep.pyx +++ b/package/MDAnalysis/coordinates/timestep.pyx @@ -938,8 +938,8 @@ cdef class Timestep: return 1.0 @dt.setter - def dt(self, new): - self.data['dt'] = new + def dt(self, new_dt): + self.data['dt'] = new_dt @dt.deleter def dt(self): @@ -966,8 +966,8 @@ cdef class Timestep: return self.dt * self.frame + offset @time.setter - def time(self, new): - self.data['time'] = new + def time(self, new_time): + self.data['time'] = new_time @time.deleter def time(self): From 19a25267560df9bf76461adde3a0033f8d990695 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Tue, 2 Sep 2025 13:54:21 -0700 Subject: [PATCH 107/114] Docs: Updated 1. API doc on multiple connections 2. Jupyter usage of `close()` for secure closure of connections --- package/MDAnalysis/coordinates/IMD.py | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 3cd46860c8b..a994cdcb2f8 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -42,6 +42,34 @@ for ts in u.trajectory: print(f'{ts.time:8.3f} {sel[0].position} {sel[0].velocity} {sel[0].force} {u.dimensions[0:3]}') +.. important:: + **Jupyter Notebook Users**: When using IMDReader in Jupyter notebooks, be aware that + **kernel restarts will not gracefully close active IMD connections**. This can leave + socket connections open, potentially preventing new connections to the same stream. + + Always use ``try/except/finally`` blocks to ensure proper cleanup: + + .. code-block:: python + + import MDAnalysis as mda + + try: + u = mda.Universe("topol.tpr", "imd://localhost:8889") + except Exception as e: + print(f"Error during connection: {e}") + else: + try: + # Your analysis code here + for ts in u.trajectory: + # Process each frame + pass + finally: + # Ensure connection is closed + u.trajectory.close() + + Always explicitly call ``u.trajectory.close()`` when finished with analysis to + ensure connection is closed properly. + Important Limitations --------------------- @@ -63,6 +91,23 @@ The IMDReader has some important limitations that are inherent in streaming data. See :class:`~MDAnalysis.coordinates.base.StreamReaderBase` for technical details. +Multiple Client Connections +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ability to establish multiple simultaneous connections to the same IMD port is +**MD engine implementation dependent**. Some simulation engines may allow multiple +clients to connect concurrently, while others may reject or fail additional connection +attempts. + +* **NAMD**: Currently supports multiple concurrent connections to the same port +* **GROMACS/LAMMPS**: Not supported by current implementation + +.. important:: + Even when multiple connections are supported by the simulation engine, each connection + receives its own independent data stream. These streams may contain different data + depending on the simulation engine's configuration, so multiple connections should + not be assumed to provide identical data streams. + .. seealso:: :class:`IMDReader` Technical details and parameter options for the reader class From b465bf1734cb6ca87da1c336eca18c326bf5eca2 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Tue, 2 Sep 2025 13:55:33 -0700 Subject: [PATCH 108/114] Cleanup: Remove timestep comment --- package/MDAnalysis/coordinates/IMD.py | 1 - 1 file changed, 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index a994cdcb2f8..f245926ebf8 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -299,7 +299,6 @@ def _load_imdframe_into_ts(self, imdf): self.ts.frame = self._frame if imdf.time is not None: self.ts.time = imdf.time - # NOTE: timestep.pyx "dt" method is suspicious bc it uses "new" keyword for a float self.ts.data["dt"] = imdf.dt self.ts.data["step"] = imdf.step if imdf.energies is not None: From 5138b54d013cd1cf00db68b85819048a5eb22137 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Tue, 2 Sep 2025 14:52:43 -0700 Subject: [PATCH 109/114] Bug Fix: Warnign message for incompatible version --- package/MDAnalysis/coordinates/IMD.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index f245926ebf8..55cad63f1bc 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -169,7 +169,7 @@ class MockIMDClient: if imdclient_version < MIN_IMDCLIENT_VERSION: warnings.warn( f"imdclient version {imdclient_version} is too old; " - f"need at least {imdclient_version}, Your installed version of " + f"need at least {MIN_IMDCLIENT_VERSION}, Your installed version of " "imdclient will NOT be used.", category=RuntimeWarning, ) From cd21d6bf4892bc8172f905816beffc1b6fff8369 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Tue, 2 Sep 2025 15:33:03 -0700 Subject: [PATCH 110/114] Refactor: simplify IMD tests with pytest monkeypatch Replace `IMDModuleStateManager` with standard pytest patterns --- .../MDAnalysisTests/coordinates/test_imd.py | 160 +++++++++--------- 1 file changed, 76 insertions(+), 84 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index 2fed2f761fd..b392570e49d 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -44,114 +44,106 @@ ) -class IMDModuleStateManager: - """Context manager to completely backup and restore imdclient/IMD module state. - - We need a custom manager because IMD changes its own state (HAS_IMDCLIENT) when it is imported - and we are going to manipulate the state of the imdclient module that IMD sees. - """ - - def __init__(self): - self.original_modules = None - self.imd_was_imported = False - - def __enter__(self): - # Backup sys.modules - self.original_modules = sys.modules.copy() - - # Check if IMD module was already imported - self.imd_was_imported = "MDAnalysis.coordinates.IMD" in sys.modules - - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - # Restore sys.modules completely first - sys.modules.clear() - sys.modules.update(self.original_modules) - - # If IMD module was originally imported, force a fresh reload to restore original state - # This ensures that HAS_IMDCLIENT and other globals are recalculated with the real imdclient - if self.imd_was_imported: - # Remove the potentially corrupted IMD module - sys.modules.pop("MDAnalysis.coordinates.IMD", None) - # Fresh import will re-evaluate all globals - import MDAnalysis.coordinates.IMD - - class TestImport: """Test imdclient import behavior and HAS_IMDCLIENT flag.""" - def _setup_mock_imdclient(self, monkeypatch, version): - """Helper method to set up mock imdclient with specified version.""" - # Remove IMD and imdclient modules to force fresh import - monkeypatch.delitem( - sys.modules, "MDAnalysis.coordinates.IMD", raising=False - ) - monkeypatch.delitem(sys.modules, "imdclient", raising=False) - + def _create_mock_imdclient(self, version="0.2.2"): + """Create a complete mock imdclient module with specified version.""" + # Module names module_name = "imdclient" - mocked_module = ModuleType(module_name) - IMDClient_module = ModuleType(f"{module_name}.IMDClient") + client_submodule_name = "IMDClient" + utils_submodule_name = "utils" + # Mock IMDClient class class MockIMDClient: pass - IMDClient_module.IMDClient = MockIMDClient - mocked_module.IMDClient = IMDClient_module - mocked_module.__version__ = version + # Main imdclient module + mock_module = ModuleType(module_name) + mock_module.__version__ = version - utils_module = ModuleType(f"{module_name}.utils") - utils_module.parse_host_port = lambda x: ("localhost", 12345) - mocked_module.utils = utils_module + # imdclient.IMDClient submodule + mock_client = ModuleType(f"{module_name}.{client_submodule_name}") + mock_client.IMDClient = MockIMDClient + mock_module.IMDClient = mock_client - monkeypatch.setitem(sys.modules, module_name, mocked_module) - monkeypatch.setitem( - sys.modules, f"{module_name}.IMDClient", IMDClient_module - ) - monkeypatch.setitem(sys.modules, f"{module_name}.utils", utils_module) + # imdclient.utils submodule + mock_utils = ModuleType(f"{module_name}.{utils_submodule_name}") + mock_utils.parse_host_port = lambda x: ("localhost", 12345) + mock_module.utils = mock_utils - return mocked_module + return mock_module, mock_client, mock_utils def test_has_minversion(self, monkeypatch): """Test that HAS_IMDCLIENT is True when imdclient >= MIN_IMDCLIENT_VERSION.""" - with IMDModuleStateManager(): - self._setup_mock_imdclient(monkeypatch, str(MIN_IMDCLIENT_VERSION)) + # Clean slate for fresh import + monkeypatch.delitem( + sys.modules, "MDAnalysis.coordinates.IMD", raising=False + ) + + # Set up mock with compatible version + mock_module, mock_client, mock_utils = self._create_mock_imdclient( + str(MIN_IMDCLIENT_VERSION) + ) + monkeypatch.setitem(sys.modules, "imdclient", mock_module) + monkeypatch.setitem(sys.modules, "imdclient.IMDClient", mock_client) + monkeypatch.setitem(sys.modules, "imdclient.utils", mock_utils) - # Import and check HAS_IMDCLIENT with compatible version - import MDAnalysis.coordinates.IMD - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + # Import should succeed with HAS_IMDCLIENT=True + import MDAnalysis.coordinates.IMD + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - assert ( - HAS_IMDCLIENT - ), f"HAS_IMDCLIENT should be True with version {MIN_IMDCLIENT_VERSION}" + assert ( + HAS_IMDCLIENT + ), f"HAS_IMDCLIENT should be True with version {MIN_IMDCLIENT_VERSION}" def test_no_minversion(self, monkeypatch): """Test that HAS_IMDCLIENT is False when imdclient version is too old.""" - with IMDModuleStateManager(): - self._setup_mock_imdclient(monkeypatch, "0.0.0") + # Clean slate for fresh import + monkeypatch.delitem( + sys.modules, "MDAnalysis.coordinates.IMD", raising=False + ) + + # Set up mock with incompatible version + mock_module, mock_client, mock_utils = self._create_mock_imdclient( + "0.0.0" + ) + monkeypatch.setitem(sys.modules, "imdclient", mock_module) + monkeypatch.setitem(sys.modules, "imdclient.IMDClient", mock_client) + monkeypatch.setitem(sys.modules, "imdclient.utils", mock_utils) - # Import and check HAS_IMDCLIENT with incompatible version - import MDAnalysis.coordinates.IMD - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + # Import should result in HAS_IMDCLIENT=False + import MDAnalysis.coordinates.IMD + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - assert ( - not HAS_IMDCLIENT - ), "HAS_IMDCLIENT should be False with version 0.0.0" + assert ( + not HAS_IMDCLIENT + ), "HAS_IMDCLIENT should be False with version 0.0.0" def test_missing_ImportError(self, monkeypatch): """Test that IMDReader raises ImportError when HAS_IMDCLIENT=False.""" - with IMDModuleStateManager(): - self._setup_mock_imdclient(monkeypatch, "0.0.0") - - # Import with incompatible version (HAS_IMDCLIENT=False) - import MDAnalysis.coordinates.IMD - from MDAnalysis.coordinates.IMD import IMDReader - - # IMDReader should raise ImportError when HAS_IMDCLIENT=False - with pytest.raises( - ImportError, match="IMDReader requires the imdclient" - ): - IMDReader("imd://localhost:12345", n_atoms=5) + # Clean slate for fresh import + monkeypatch.delitem( + sys.modules, "MDAnalysis.coordinates.IMD", raising=False + ) + + # Set up mock with incompatible version (will set HAS_IMDCLIENT=False) + mock_main, mock_client, mock_utils = self._create_mock_imdclient( + "0.0.0" + ) + monkeypatch.setitem(sys.modules, "imdclient", mock_main) + monkeypatch.setitem(sys.modules, "imdclient.IMDClient", mock_client) + monkeypatch.setitem(sys.modules, "imdclient.utils", mock_utils) + + # Import with HAS_IMDCLIENT=False + import MDAnalysis.coordinates.IMD + from MDAnalysis.coordinates.IMD import IMDReader + + # IMDReader should raise ImportError + with pytest.raises( + ImportError, match="IMDReader requires the imdclient" + ): + IMDReader("imd://localhost:12345", n_atoms=5) @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") From feb6c9dd96b99b5dfa52159163ab206e3bc62a45 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Mon, 8 Sep 2025 08:05:40 -0700 Subject: [PATCH 111/114] Revert "Refactor: simplify IMD tests with pytest monkeypatch" This reverts commit cd21d6bf4892bc8172f905816beffc1b6fff8369. --- .../MDAnalysisTests/coordinates/test_imd.py | 160 +++++++++--------- 1 file changed, 84 insertions(+), 76 deletions(-) diff --git a/testsuite/MDAnalysisTests/coordinates/test_imd.py b/testsuite/MDAnalysisTests/coordinates/test_imd.py index b392570e49d..2fed2f761fd 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_imd.py +++ b/testsuite/MDAnalysisTests/coordinates/test_imd.py @@ -44,106 +44,114 @@ ) +class IMDModuleStateManager: + """Context manager to completely backup and restore imdclient/IMD module state. + + We need a custom manager because IMD changes its own state (HAS_IMDCLIENT) when it is imported + and we are going to manipulate the state of the imdclient module that IMD sees. + """ + + def __init__(self): + self.original_modules = None + self.imd_was_imported = False + + def __enter__(self): + # Backup sys.modules + self.original_modules = sys.modules.copy() + + # Check if IMD module was already imported + self.imd_was_imported = "MDAnalysis.coordinates.IMD" in sys.modules + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Restore sys.modules completely first + sys.modules.clear() + sys.modules.update(self.original_modules) + + # If IMD module was originally imported, force a fresh reload to restore original state + # This ensures that HAS_IMDCLIENT and other globals are recalculated with the real imdclient + if self.imd_was_imported: + # Remove the potentially corrupted IMD module + sys.modules.pop("MDAnalysis.coordinates.IMD", None) + # Fresh import will re-evaluate all globals + import MDAnalysis.coordinates.IMD + + class TestImport: """Test imdclient import behavior and HAS_IMDCLIENT flag.""" - def _create_mock_imdclient(self, version="0.2.2"): - """Create a complete mock imdclient module with specified version.""" - # Module names + def _setup_mock_imdclient(self, monkeypatch, version): + """Helper method to set up mock imdclient with specified version.""" + # Remove IMD and imdclient modules to force fresh import + monkeypatch.delitem( + sys.modules, "MDAnalysis.coordinates.IMD", raising=False + ) + monkeypatch.delitem(sys.modules, "imdclient", raising=False) + module_name = "imdclient" - client_submodule_name = "IMDClient" - utils_submodule_name = "utils" + mocked_module = ModuleType(module_name) + IMDClient_module = ModuleType(f"{module_name}.IMDClient") - # Mock IMDClient class class MockIMDClient: pass - # Main imdclient module - mock_module = ModuleType(module_name) - mock_module.__version__ = version + IMDClient_module.IMDClient = MockIMDClient + mocked_module.IMDClient = IMDClient_module + mocked_module.__version__ = version - # imdclient.IMDClient submodule - mock_client = ModuleType(f"{module_name}.{client_submodule_name}") - mock_client.IMDClient = MockIMDClient - mock_module.IMDClient = mock_client + utils_module = ModuleType(f"{module_name}.utils") + utils_module.parse_host_port = lambda x: ("localhost", 12345) + mocked_module.utils = utils_module - # imdclient.utils submodule - mock_utils = ModuleType(f"{module_name}.{utils_submodule_name}") - mock_utils.parse_host_port = lambda x: ("localhost", 12345) - mock_module.utils = mock_utils + monkeypatch.setitem(sys.modules, module_name, mocked_module) + monkeypatch.setitem( + sys.modules, f"{module_name}.IMDClient", IMDClient_module + ) + monkeypatch.setitem(sys.modules, f"{module_name}.utils", utils_module) - return mock_module, mock_client, mock_utils + return mocked_module def test_has_minversion(self, monkeypatch): """Test that HAS_IMDCLIENT is True when imdclient >= MIN_IMDCLIENT_VERSION.""" - # Clean slate for fresh import - monkeypatch.delitem( - sys.modules, "MDAnalysis.coordinates.IMD", raising=False - ) - - # Set up mock with compatible version - mock_module, mock_client, mock_utils = self._create_mock_imdclient( - str(MIN_IMDCLIENT_VERSION) - ) - monkeypatch.setitem(sys.modules, "imdclient", mock_module) - monkeypatch.setitem(sys.modules, "imdclient.IMDClient", mock_client) - monkeypatch.setitem(sys.modules, "imdclient.utils", mock_utils) + with IMDModuleStateManager(): + self._setup_mock_imdclient(monkeypatch, str(MIN_IMDCLIENT_VERSION)) - # Import should succeed with HAS_IMDCLIENT=True - import MDAnalysis.coordinates.IMD - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + # Import and check HAS_IMDCLIENT with compatible version + import MDAnalysis.coordinates.IMD + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - assert ( - HAS_IMDCLIENT - ), f"HAS_IMDCLIENT should be True with version {MIN_IMDCLIENT_VERSION}" + assert ( + HAS_IMDCLIENT + ), f"HAS_IMDCLIENT should be True with version {MIN_IMDCLIENT_VERSION}" def test_no_minversion(self, monkeypatch): """Test that HAS_IMDCLIENT is False when imdclient version is too old.""" - # Clean slate for fresh import - monkeypatch.delitem( - sys.modules, "MDAnalysis.coordinates.IMD", raising=False - ) - - # Set up mock with incompatible version - mock_module, mock_client, mock_utils = self._create_mock_imdclient( - "0.0.0" - ) - monkeypatch.setitem(sys.modules, "imdclient", mock_module) - monkeypatch.setitem(sys.modules, "imdclient.IMDClient", mock_client) - monkeypatch.setitem(sys.modules, "imdclient.utils", mock_utils) + with IMDModuleStateManager(): + self._setup_mock_imdclient(monkeypatch, "0.0.0") - # Import should result in HAS_IMDCLIENT=False - import MDAnalysis.coordinates.IMD - from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT + # Import and check HAS_IMDCLIENT with incompatible version + import MDAnalysis.coordinates.IMD + from MDAnalysis.coordinates.IMD import HAS_IMDCLIENT - assert ( - not HAS_IMDCLIENT - ), "HAS_IMDCLIENT should be False with version 0.0.0" + assert ( + not HAS_IMDCLIENT + ), "HAS_IMDCLIENT should be False with version 0.0.0" def test_missing_ImportError(self, monkeypatch): """Test that IMDReader raises ImportError when HAS_IMDCLIENT=False.""" - # Clean slate for fresh import - monkeypatch.delitem( - sys.modules, "MDAnalysis.coordinates.IMD", raising=False - ) - - # Set up mock with incompatible version (will set HAS_IMDCLIENT=False) - mock_main, mock_client, mock_utils = self._create_mock_imdclient( - "0.0.0" - ) - monkeypatch.setitem(sys.modules, "imdclient", mock_main) - monkeypatch.setitem(sys.modules, "imdclient.IMDClient", mock_client) - monkeypatch.setitem(sys.modules, "imdclient.utils", mock_utils) - - # Import with HAS_IMDCLIENT=False - import MDAnalysis.coordinates.IMD - from MDAnalysis.coordinates.IMD import IMDReader - - # IMDReader should raise ImportError - with pytest.raises( - ImportError, match="IMDReader requires the imdclient" - ): - IMDReader("imd://localhost:12345", n_atoms=5) + with IMDModuleStateManager(): + self._setup_mock_imdclient(monkeypatch, "0.0.0") + + # Import with incompatible version (HAS_IMDCLIENT=False) + import MDAnalysis.coordinates.IMD + from MDAnalysis.coordinates.IMD import IMDReader + + # IMDReader should raise ImportError when HAS_IMDCLIENT=False + with pytest.raises( + ImportError, match="IMDReader requires the imdclient" + ): + IMDReader("imd://localhost:12345", n_atoms=5) @pytest.mark.skipif(not HAS_IMDCLIENT, reason="IMDClient not installed") From 35fcc8e734540aef388c6bbb47bcaf4196e25508 Mon Sep 17 00:00:00 2001 From: Oliver Beckstein Date: Mon, 8 Sep 2025 10:33:49 -0700 Subject: [PATCH 112/114] IMDReader docs: reorganized and reduced multiple connection details - move details on which engines support multiple connections to imdclient docs (only update one place) and link - rearranged warning/seealso boxes --- package/MDAnalysis/coordinates/IMD.py | 40 +++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/package/MDAnalysis/coordinates/IMD.py b/package/MDAnalysis/coordinates/IMD.py index 55cad63f1bc..4945d43adfc 100644 --- a/package/MDAnalysis/coordinates/IMD.py +++ b/package/MDAnalysis/coordinates/IMD.py @@ -19,6 +19,18 @@ monitoring and analysis of ongoing simulations. It uses the `imdclient package `_ (dependency) to implement the IMDv3 protocol and manage the socket connection and data parsing. +.. seealso:: + :class:`IMDReader` + Technical details and parameter options for the reader class + + `imdclient documentation `_ + Complete documentation for the IMDClient package + + `IMDClient GitHub repository `_ + Source code and development resources + +.. _`imdclient simulation engine docs`: https://imdclient.readthedocs.io/en/latest/usage.html + Usage Example ------------- @@ -73,6 +85,9 @@ Important Limitations --------------------- +.. warning:: + The IMDReader has some important limitations that are inherent in streaming data. + Since IMD streams data in real-time from a running simulation, it has fundamental constraints that differ from traditional trajectory readers: @@ -87,39 +102,24 @@ * **Single client connection**: Only one reader can connect to an IMD stream at a time * **No trajectory Writing**: Complimentary IMD Writer class is not available for streaming data -.. warning:: - The IMDReader has some important limitations that are inherent in streaming data. - See :class:`~MDAnalysis.coordinates.base.StreamReaderBase` for technical details. +.. seealso:: + See :class:`~MDAnalysis.coordinates.base.StreamReaderBase` for technical details. Multiple Client Connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +--------------------------- The ability to establish multiple simultaneous connections to the same IMD port is **MD engine implementation dependent**. Some simulation engines may allow multiple clients to connect concurrently, while others may reject or fail additional connection attempts. -* **NAMD**: Currently supports multiple concurrent connections to the same port -* **GROMACS/LAMMPS**: Not supported by current implementation +See the `imdclient simulation engine docs`_ for further details. .. important:: Even when multiple connections are supported by the simulation engine, each connection receives its own independent data stream. These streams may contain different data depending on the simulation engine's configuration, so multiple connections should - not be assumed to provide identical data streams. - -.. seealso:: - :class:`IMDReader` - Technical details and parameter options for the reader class - - `imdclient documentation `_ - Complete documentation for the IMDClient package - - `IMDClient GitHub repository `_ - Source code and development resources - -.. _`imdclient simulation engine docs`: https://imdclient.readthedocs.io/en/latest/usage.html - + not be assumed to provide identical data streams. Classes ------- From f2a505009ecd3c991d7ebd2e1df46538ce4686ce Mon Sep 17 00:00:00 2001 From: Oliver Beckstein Date: Fri, 19 Sep 2025 11:31:35 -0700 Subject: [PATCH 113/114] Update package/pyproject.toml Co-authored-by: Irfan Alibay --- package/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/package/pyproject.toml b/package/pyproject.toml index 0da719731a4..bb9d9d48929 100644 --- a/package/pyproject.toml +++ b/package/pyproject.toml @@ -77,7 +77,6 @@ extra_formats = [ "gsd>3.0.0", "rdkit>=2022.09.1", "imdclient>=0.2.2", - ] analysis = [ "biopython>=1.80", From ea85bb510f684ef1f6048ad0a50219665298cbb3 Mon Sep 17 00:00:00 2001 From: Amruthesh Thirumalaiswamy Date: Fri, 19 Sep 2025 15:12:47 -0700 Subject: [PATCH 114/114] Chore: Minor cleanup 1. Alphabetical order for `imclcient` as optional dependency 2. Changed order in `CHANGELOG` --- .github/actions/setup-deps/action.yaml | 6 +++--- package/CHANGELOG | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/actions/setup-deps/action.yaml b/.github/actions/setup-deps/action.yaml index 0a46c7e5da8..7523214b8fa 100644 --- a/.github/actions/setup-deps/action.yaml +++ b/.github/actions/setup-deps/action.yaml @@ -64,6 +64,8 @@ inputs: default: 'h5py>=2.10' hole2: default: 'hole2' + imdclient: + default: 'imdclient>=0.2.2' joblib: default: 'joblib>=0.12' netcdf4: @@ -82,8 +84,6 @@ inputs: default: 'seaborn>=0.7.0' tidynamics: default: 'tidynamics>=1.0.0' - imdclient: - default: 'imdclient>=0.2.2' # pip-installed min dependencies coverage: default: 'coverage' @@ -140,6 +140,7 @@ runs: ${{ inputs.gsd }} ${{ inputs.h5py }} ${{ inputs.hole2 }} + ${{ inputs.imdclient }} ${{ inputs.joblib }} ${{ inputs.netcdf4 }} ${{ inputs.networkx }} @@ -149,7 +150,6 @@ runs: ${{ inputs.scikit-learn }} ${{ inputs.seaborn }} ${{ inputs.tidynamics }} - ${{ inputs.imdclient }} run: | # setup full variable diff --git a/package/CHANGELOG b/package/CHANGELOG index ba1975c4a22..ebaa2902bd6 100644 --- a/package/CHANGELOG +++ b/package/CHANGELOG @@ -42,6 +42,11 @@ Fixes directly passing them. (Issue #3520, PR #5006) Enhancements + * Added support for reading and processing streamed data in `coordinates.base` + with new `StreamFrameIteratorSliced` and `StreamReaderBase` (Issue #4827, PR #4923) + * New coordinate reader: Added `IMDReader` for reading real-time streamed + molecular dynamics simulation data using the IMDv3 protocol - requires + `imdclient` package (Issue #4827, PR #4923) * Added capability to calculate MSD from frames with irregular (non-linear) time spacing in analysis.msd.EinsteinMSD with keyword argument `non_linear=True` (Issue #5028, PR #5066) @@ -70,12 +75,7 @@ Enhancements so that it gets passed through from the calling functions and classes (PR #5038) * Moved distopia checking function to common import location in - MDAnalysisTest.util (PR #5038) - * Added support for reading and processing streamed data in `coordinates.base` - with new `StreamFrameIteratorSliced` and `StreamReaderBase` (Issue #4827, PR #4923) - * New coordinate reader: Added `IMDReader` for reading real-time streamed - molecular dynamics simulation data using the IMDv3 protocol - requires - `imdclient` package (Issue #4827, PR #4923) + MDAnalysisTest.util (PR #5038) * Enables parallelization for `analysis.polymer.PersistenceLength` (Issue #4671, PR #5074)