From cec61d04d0118150355038683eb69d0b45a6c4f5 Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Wed, 3 Sep 2025 17:04:37 -0400 Subject: [PATCH 1/5] Start smurf streams within try block Otherwise a SIGINT might be timed just right to start most of the streams, but never stop them. --- src/sorunlib/seq.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sorunlib/seq.py b/src/sorunlib/seq.py index b43988a2..197e68d1 100644 --- a/src/sorunlib/seq.py +++ b/src/sorunlib/seq.py @@ -66,10 +66,10 @@ def scan(description, stop_time, width, az_drift=0, tag=None, subtype=None, acu = run.CLIENTS['acu'] - # Enable SMuRF streams - run.smurf.stream('on', subtype=subtype, tag=tag) - try: + # Enable SMuRF streams + run.smurf.stream('on', subtype=subtype, tag=tag) + # Grab current telescope position resp = acu.monitor.status() az = resp.session['data']['StatusDetailed']['Azimuth current position'] @@ -110,10 +110,10 @@ def el_nod(el1, el2, num=5, pause=5): """ acu = run.CLIENTS['acu'] - # Enable SMuRF streams - run.smurf.stream('on', subtype='cal', tag='el_nods') - try: + # Enable SMuRF streams + run.smurf.stream('on', subtype='cal', tag='el_nods') + # Grab current telescope position resp = acu.monitor.status() init_az = resp.session['data']['StatusDetailed']['Azimuth current position'] From 1f4dbf157e3d7e42221fdc57a451b99bc09777d4 Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Tue, 9 Sep 2025 15:36:14 -0400 Subject: [PATCH 2/5] Add print statement before commanding ACU to stop --- src/sorunlib/seq.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sorunlib/seq.py b/src/sorunlib/seq.py index 197e68d1..f4f960d7 100644 --- a/src/sorunlib/seq.py +++ b/src/sorunlib/seq.py @@ -26,6 +26,7 @@ def _stop_scan(): # Stop motion acu.generate_scan.stop() + print("Waiting for telescope motion to stop.") resp = acu.generate_scan.wait(timeout=OP_TIMEOUT) check_response(acu, resp) print("Scan finished.") From 02bf663efd8f0fb6db77300b512b242841127300 Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Tue, 9 Sep 2025 11:47:37 -0400 Subject: [PATCH 3/5] Handle SIGTERM like SIGINT and raise KeyboardInterrupt --- src/sorunlib/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/sorunlib/__init__.py b/src/sorunlib/__init__.py index c07ec0e9..ce0b9ee1 100644 --- a/src/sorunlib/__init__.py +++ b/src/sorunlib/__init__.py @@ -1,3 +1,5 @@ +import signal + from . import acu, hwp, seq, smurf, stimulator, wiregrid from .commands import wait_until @@ -27,6 +29,15 @@ def initialize(test_mode=False): "wait_until", "initialize"] + +# Treat SIGTERM like SIGINT and raise an exception +def term_handler(sig, frame): + raise KeyboardInterrupt + + +signal.signal(signal.SIGTERM, term_handler) + + # Define the variable '__version__': # This has the closest behavior to versioneer that I could find # https://github.com/maresb/hatch-vcs-footgun-example From 39d6e2476e5a879fea6d25db22f00a2d7a68575c Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Tue, 9 Sep 2025 15:49:52 -0400 Subject: [PATCH 4/5] Move SMuRF stream stop function into internal module This simple wrapper was created to help with error handling and testing. We had similar code elsewhere, so we replace direct uses of run.smurf.stream('off') with this wrapper. --- src/sorunlib/_internal.py | 12 ++++++++++++ src/sorunlib/hwp.py | 6 +++--- src/sorunlib/seq.py | 14 +++----------- src/sorunlib/stimulator.py | 12 +++--------- src/sorunlib/wiregrid.py | 16 ++++++++-------- 5 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/sorunlib/_internal.py b/src/sorunlib/_internal.py index c59c8768..ba549fce 100644 --- a/src/sorunlib/_internal.py +++ b/src/sorunlib/_internal.py @@ -8,6 +8,7 @@ import time import ocs +import sorunlib as run from sorunlib.commands import _timestamp_to_utc_datetime @@ -180,3 +181,14 @@ def monitor_process(client, operation, stop_time, check_interval=10): # Recompute diff diff = _seconds_until_target(stop_time) + + +def stop_smurfs(): + """Simple wrapper to shutdown all SMuRF systems and handle any errors that + occur. + + """ + try: + run.smurf.stream('off') + except RuntimeError as e: + print(f"Caught error while shutting down SMuRF streams: {e}") diff --git a/src/sorunlib/hwp.py b/src/sorunlib/hwp.py index 7d5d495e..04bf2ddb 100644 --- a/src/sorunlib/hwp.py +++ b/src/sorunlib/hwp.py @@ -1,5 +1,5 @@ import sorunlib as run -from sorunlib._internal import check_response +from sorunlib._internal import check_response, stop_smurfs def _get_direction(): @@ -74,7 +74,7 @@ def spin_up(freq): check_response(hwp, resp) run.hwp.set_freq(freq=freq, timeout=1800) finally: - run.smurf.stream('off') + stop_smurfs() def spin_down(active=True, brake_voltage=None): @@ -97,7 +97,7 @@ def spin_down(active=True, brake_voltage=None): resp = hwp.disable_driver_board() check_response(hwp, resp) finally: - run.smurf.stream('off') + stop_smurfs() def stop(active=True, brake_voltage=None): diff --git a/src/sorunlib/seq.py b/src/sorunlib/seq.py index f4f960d7..37b94c49 100644 --- a/src/sorunlib/seq.py +++ b/src/sorunlib/seq.py @@ -4,25 +4,17 @@ import sorunlib as run from sorunlib.commands import _timestamp_to_utc_datetime -from sorunlib._internal import check_response, check_started, monitor_process +from sorunlib._internal import check_response, check_started, monitor_process, stop_smurfs OP_TIMEOUT = 60 -def _stop_smurfs(): - # Stop SMuRF streams - try: - run.smurf.stream('off') - except RuntimeError as e: - print(f"Caught error while shutting down SMuRF streams: {e}") - - def _stop_scan(): acu = run.CLIENTS['acu'] print("Stopping scan.") - _stop_smurfs() + stop_smurfs() # Stop motion acu.generate_scan.stop() @@ -130,4 +122,4 @@ def el_nod(el1, el2, num=5, pause=5): # Return to initial position run.acu.move_to(az=init_az, el=init_el) finally: - _stop_smurfs() + stop_smurfs() diff --git a/src/sorunlib/stimulator.py b/src/sorunlib/stimulator.py index bd9bcc25..c9b72df8 100644 --- a/src/sorunlib/stimulator.py +++ b/src/sorunlib/stimulator.py @@ -1,6 +1,6 @@ import time import sorunlib as run -from sorunlib._internal import check_response +from sorunlib._internal import check_response, stop_smurfs ID_SHUTTER = 1 @@ -99,10 +99,7 @@ def calibrate_tau(duration_step=10, time.sleep(duration_step) finally: - try: - run.smurf.stream('off') - except RuntimeError as e: - print(f"Caught error while shutting down SMuRF streams: {e}") + stop_smurfs() if stop: _stop() @@ -147,10 +144,7 @@ def calibrate_gain(duration=60, speed_rpm=90, # Data taking time.sleep(duration) finally: - try: - run.smurf.stream('off') - except RuntimeError as e: - print(f"Caught error while shutting down SMuRF streams: {e}") + stop_smurfs() if stop: _stop() diff --git a/src/sorunlib/wiregrid.py b/src/sorunlib/wiregrid.py index f8b993c6..b98cead6 100644 --- a/src/sorunlib/wiregrid.py +++ b/src/sorunlib/wiregrid.py @@ -1,7 +1,7 @@ import time import sorunlib as run -from sorunlib._internal import check_response, check_running +from sorunlib._internal import check_response, check_running, stop_smurfs EL_DIFF_THRESHOLD = 0.5 # deg diff from target that its ok to run calibration BORESIGHT_DIFF_THRESHOLD = 0.5 # deg @@ -322,7 +322,7 @@ def calibrate(continuous=False, elevation_check=True, boresight_check=True, eject() finally: # Stop SMuRF streams - run.smurf.stream('off') + stop_smurfs() def time_constant(num_repeats=1): @@ -381,7 +381,7 @@ def time_constant(num_repeats=1): insert() time.sleep(5) finally: - run.smurf.stream('off') + stop_smurfs() for i in range(num_repeats): if current_hwp_direction == 'ccw': @@ -404,7 +404,7 @@ def time_constant(num_repeats=1): # Run stepwise rotation rotate(continuous=False) finally: - run.smurf.stream('off') + stop_smurfs() # Stop the HWP while streaming try: @@ -413,7 +413,7 @@ def time_constant(num_repeats=1): run.smurf.stream('on', tag=stream_tag, subtype='cal') run.hwp.stop(active=True) finally: - run.smurf.stream('off') + stop_smurfs() # Reverse the HWP while streaming try: @@ -430,7 +430,7 @@ def time_constant(num_repeats=1): run.hwp.set_freq(freq=-2.0) current_hwp_direction = target_hwp_direction finally: - run.smurf.stream('off') + stop_smurfs() # Run stepwise rotation after changing the HWP rotation try: @@ -441,7 +441,7 @@ def time_constant(num_repeats=1): # Run stepwise rotation rotate(continuous=False) finally: - run.smurf.stream('off') + stop_smurfs() # Bias step (the wire grid is on the window) # After changing the HWP rotation @@ -458,7 +458,7 @@ def time_constant(num_repeats=1): eject() time.sleep(5) finally: - run.smurf.stream('off') + stop_smurfs() # Bias step (the wire grid is off the window) bs_tag = 'wiregrid, wg_time_constant, wg_ejected, ' + \ From 04c527c3ff6fe21011583ebaa59f06f9d29cbaaf Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Tue, 9 Sep 2025 15:51:24 -0400 Subject: [PATCH 5/5] Create a decorator to add signal handling during shutdown This can be used to protect shutdown functions that performing cleanup before exiting. This will prevent accidental interruptions of cleanup code, which in the past has left SMuRF streams or ACU motion on. --- src/sorunlib/_internal.py | 28 ++++++++++++++++++++++++++++ src/sorunlib/seq.py | 3 ++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/sorunlib/_internal.py b/src/sorunlib/_internal.py index ba549fce..94e3d468 100644 --- a/src/sorunlib/_internal.py +++ b/src/sorunlib/_internal.py @@ -5,8 +5,11 @@ """ import datetime as dt +import signal import time +from functools import wraps + import ocs import sorunlib as run @@ -183,6 +186,31 @@ def monitor_process(client, operation, stop_time, check_interval=10): diff = _seconds_until_target(stop_time) +def protect_shutdown(f): + """Decorator to install temporary signal handlers while operations required + to safely shutdown are handled. + + This will catch and print the caught signals to ``stdout`` while shutdown + is happening. Currently handles only ``SIGINT`` and ``SIGTERM``. + + """ + @wraps(f) + def wrapper(*args, **kwds): + def handler(sig, frame): + print(f'Caught {signal.Signals(sig).name} during shutdown.') + + int_handler = signal.signal(signal.SIGINT, handler) + term_handler = signal.signal(signal.SIGTERM, handler) + + result = f(*args, **kwds) + + signal.signal(signal.SIGINT, int_handler) + signal.signal(signal.SIGTERM, term_handler) + return result + return wrapper + + +@protect_shutdown def stop_smurfs(): """Simple wrapper to shutdown all SMuRF systems and handle any errors that occur. diff --git a/src/sorunlib/seq.py b/src/sorunlib/seq.py index 37b94c49..e039ff4a 100644 --- a/src/sorunlib/seq.py +++ b/src/sorunlib/seq.py @@ -4,12 +4,13 @@ import sorunlib as run from sorunlib.commands import _timestamp_to_utc_datetime -from sorunlib._internal import check_response, check_started, monitor_process, stop_smurfs +from sorunlib._internal import check_response, check_started, monitor_process, protect_shutdown, stop_smurfs OP_TIMEOUT = 60 +@protect_shutdown def _stop_scan(): acu = run.CLIENTS['acu']