Hyperion supervisor implementation

docs/developer/hyperion/hyperion-blueapi.rst

@@ -23,7 +23,7 @@ Hyperion on BlueAPI consists of two components:
 Deployment
 ----------
 
-``hyperion-blueapi`` is automatically available in a standard Hyperion deployment.
+``hyperion-blueapi`` and ``hyperion-supervisor`` are automatically available in a standard Hyperion deployment.
 
 Launching
 ---------
@@ -33,3 +33,33 @@ Launching
 ::
 
     ./run_hyperion.sh --beamline=i03 --dev --blueapi
+
+
+``hyperion-supervisor`` can be launched using the ``run_hyperion.sh`` script, using the ``--supervisor`` option:
+
+::
+
+    ./run_hyperion.sh --beamline=i03 --dev --supervisor
+
+Configuration
+-------------
+
+Configuration of ``hyperion-blueapi`` and ``hyperion-supervisor`` is done via standard BlueAPI .yaml configuration files.
+Basic configuration files for i03 are supplied as follows in ``src/mx_bluesky/hyperion``.
+
+.. csv-table:: hyperion-blueapi configuration files
+    :widths: auto
+    :header: "File", "Description"
+
+    "blueapi_config.yaml", "Defines beamline device module and blueapi plans to be exported, BlueAPI REST to listen on, Stomp and graylog servers."
+
+
+.. csv-table:: hyperion-supervisor configuration files
+    :widths: auto
+    :header: "File", "Description"
+
+    "supervisor/client_config.yaml", "Tells the supervisor how to communicate with hyperion-blueapi, specifying the REST endpoint and stomp server."
+    "supervisor/supervisor_config.yaml", "Configures the internal blueapi context with a minimal beamline module containing the baton device and the graylog endpoint."
+
+When these are  deployed in kubernetes it is anticipated that these will be provided by specifying
+directly in the values.yaml which will be expanded by the base helmcharts at deployment time.

pyproject.toml

@@ -48,7 +48,7 @@ dependencies = [
     "ophyd >= 1.10.5",
     "ophyd-async >= 0.14.0",
     "bluesky >= 1.14.6",
-    "dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@a946a46ffd418ea25fbae9134954659cfa223d6f",
+    "dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@a6d784617a42cb5fc2139d361fe0323532fffb66",
 ]
 
 

run_hyperion.sh

@@ -9,6 +9,11 @@ MODE=gda
 
 CONFIG_DIR=`dirname $0`/src/mx_bluesky/hyperion
 BLUEAPI_CONFIG=$CONFIG_DIR/blueapi_config.yaml
+SUPERVISOR_CONFIG=$CONFIG_DIR/supervisor/supervisor_config.yaml
+CLIENT_CONFIG=$CONFIG_DIR/supervisor/client_config.yaml
+DO_CALLBACKS=1
+HEALTHCHECK_PORT=5005
+CALLBACK_WATCHDOG_PORT=5005
 
 for option in "$@"; do
     case $option in
@@ -26,12 +31,19 @@ for option in "$@"; do
         --dev)
             IN_DEV=true
             BLUEAPI_CONFIG=$CONFIG_DIR/blueapi_dev_config.yaml
+            SUPERVISOR_CONFIG=$CONFIG_DIR/supervisor/supervisor_dev_config.yaml
             ;;
         --udc)
             MODE=udc
             ;;
         --blueapi)
             MODE=blueapi
+            CALLBACK_WATCHDOG_PORT=5006
+            ;;
+        --supervisor)
+            MODE=supervisor
+            DO_CALLBACKS=0
+            HEALTHCHECK_PORT=5006
             ;;
         --help|--info|--h)
             source .venv/bin/activate
@@ -48,6 +60,8 @@ Options:
   --dev                   Enable dev mode to run from a local workspace on a development machine.
   --udc                   Start hyperion in UDC mode instead of taking commands from GDA
   --blueapi               Start hyperion in blueapi mode instead of taking commands from GDA
+  --supervisor            Start hyperion in supervisor mode, taking commands from Agamemnon and feeding them to
+                          an instance running in blueapi mode.
   --help                  This help
 
 By default this script will start an Hyperion server unless the --no-start flag is specified.
@@ -62,11 +76,18 @@ END
 done
 
 kill_active_apps () {
-    echo "Killing active instances of hyperion and hyperion-callbacks..."
-    pkill -e -f "python.*hyperion"
-    pkill -e -f "SCREEN.*hyperion"
-    blueapi controller stop 2>/dev/null
-    echo "done."
+    if [ $MODE = "supervisor" ]; then
+      # supervisor mode kills only supervisor
+      echo "Killing active instances of hyperion supervisor..."
+      pkill -e -f "mx-bluesky/.venv/bin/python .*--mode supervisor"
+    else
+      echo "Killing active instances of hyperion-blueapi"
+      pkill -e -f "python .*mx-bluesky/.venv/bin/blueapi .*serve"
+      echo "Killing vanilla hyperion instances"
+      pkill -e -f "mx-bluesky/.venv/bin/python .*--mode (gda|udc)"
+      echo "Killing hyperion-callbacks"
+      pkill -e -f "mx-bluesky/.venv/bin/python .*hyperion-callbacks"
+    fi
 }
 
 check_user () {
@@ -99,7 +120,7 @@ if [[ $START == 1 ]]; then
         check_user
         ISPYB_CONFIG_PATH="/dls_sw/dasc/mariadb/credentials/ispyb-hyperion-${BEAMLINE}.cfg"
     else
-        ISPYB_CONFIG_PATH="$RELATIVE_SCRIPT_DIR/tests/test_data/ispyb_test_credentials.cfg"
+        ISPYB_CONFIG_PATH="$RELATIVE_SCRIPT_DIR/tests/test_data/ispyb-test-credentials.cfg"
         ZOCALO_CONFIG="$RELATIVE_SCRIPT_DIR/tests/test_data/zocalo-test-configuration.yaml"
         export ZOCALO_CONFIG
     fi
@@ -121,17 +142,24 @@ if [[ $START == 1 ]]; then
     fi
     echo "$(date) Logging to $LOG_DIR"
     export LOG_DIR
-    mkdir -p $LOG_DIR
-    start_log_path=$LOG_DIR/start_log.log
+    mkdir -p "$LOG_DIR"
+    if [ $MODE = "supervisor" ]; then
+      start_log_path=$LOG_DIR/supervisor_start_log.log
+    else
+      start_log_path=$LOG_DIR/start_log.log
+    fi
     callback_start_log_path=$LOG_DIR/callback_start_log.log
 
     source .venv/bin/activate
 
     declare -A h_and_cb_args=( ["IN_DEV"]="$IN_DEV" )
     declare -A h_and_cb_arg_strings=( ["IN_DEV"]="--dev" )
 
-    h_commands="--mode $MODE"
-    cb_commands=()
+    h_commands="--mode $MODE "
+    cb_commands="--watchdog-port $CALLBACK_WATCHDOG_PORT "
+    if [ $MODE = "supervisor" ]; then
+      h_commands+="--client-config ${CLIENT_CONFIG} --supervisor-config ${SUPERVISOR_CONFIG} "
+    fi
     for i in "${!h_and_cb_args[@]}"
     do
         if [ "${h_and_cb_args[$i]}" != false ]; then 
@@ -146,18 +174,20 @@ if [[ $START == 1 ]]; then
       blueapi --config $BLUEAPI_CONFIG serve > $start_log_path 2>&1 &
       HEALTHCHECK_ENDPOINT="healthz"
     else
-      echo "Starting hyperion with hyperion $h_commands, start_log is $start_log_path"
+      echo "Starting hyperion in mode $MODE with hyperion $h_commands, start_log is $start_log_path"
       hyperion `echo $h_commands;`>$start_log_path  2>&1 &
       HEALTHCHECK_ENDPOINT="status"
     fi
-    echo "Starting hyperion-callbacks with hyperion-callbacks $cb_commands, start_log is $callback_start_log_path"
-    hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 &
+    if [[ $DO_CALLBACKS == 1 ]]; then
+      echo "Starting hyperion-callbacks with hyperion-callbacks $cb_commands, start_log is $callback_start_log_path"
+      hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 &
+    fi
     echo "$(date) Waiting for Hyperion to start"
 
     for i in {1..30}
     do
         echo "$(date)"
-        curl --head -X GET http://localhost:5005/$HEALTHCHECK_ENDPOINT >/dev/null
+        curl --head -X GET http://localhost:$HEALTHCHECK_PORT/$HEALTHCHECK_ENDPOINT >/dev/null
         ret_value=$?
         if [ $ret_value -ne 0 ]; then
             sleep 1

src/mx_bluesky/Getting started.ipynb

@@ -52,6 +52,7 @@
     "import importlib\n",
     "\n",
     "from dodal.utils import collect_factories\n",
+    "\n",
     "beamline = \"i02_2\"\n",
     "module_name = f\"dodal.beamlines.{beamline}\"\n",
     "beamline_module = importlib.import_module(module_name)\n",

src/mx_bluesky/hyperion/__main__.py

@@ -2,9 +2,11 @@
 import signal
 import threading
 from dataclasses import asdict
+from pathlib import Path
 from sys import argv
 from traceback import format_exception
 
+from blueapi.config import ApplicationConfig, ConfigLoader
 from blueapi.core import BlueskyContext
 from flask import Flask, request
 from flask_restful import Api, Resource
@@ -43,6 +45,7 @@
     StatusAndMessage,
     make_error_status_and_message,
 )
+from mx_bluesky.hyperion.supervisor import SupervisorRunner
 from mx_bluesky.hyperion.utils.context import setup_context
 
 
@@ -153,7 +156,11 @@ def create_app(runner: GDARunner, test_config=None) -> Flask:
 def initialise_globals(args: HyperionArgs):
     """Do all early main low-level application initialisation."""
     do_default_logging_setup(
-        CONST.LOG_FILE_NAME, CONST.GRAYLOG_PORT, dev_mode=args.dev_mode
+        CONST.SUPERVISOR_LOG_FILE_NAME
+        if args.mode == HyperionMode.SUPERVISOR
+        else CONST.LOG_FILE_NAME,
+        CONST.GRAYLOG_PORT,
+        dev_mode=args.dev_mode,
     )
     LOGGER.info(f"Hyperion launched with args:{argv}")
     alerting.set_alerting_service(LoggingAlertService(CONST.GRAYLOG_STREAM_ID))
@@ -184,13 +191,32 @@ def main():
         case HyperionMode.UDC:
             context = setup_context(dev_mode=args.dev_mode)
             plan_runner = InProcessRunner(context, args.dev_mode)
-            create_server_for_udc(plan_runner)
+            create_server_for_udc(plan_runner, HyperionConstants.HYPERION_PORT)
             _register_sigterm_handler(plan_runner)
             run_forever(plan_runner)
         case HyperionMode.SUPERVISOR:
-            raise RuntimeError(
-                "Supervisor mode not supported yet see https://github.com/DiamondLightSource/mx-bluesky/issues/1365"
-            )
+            if not args.client_config:
+                raise RuntimeError(
+                    "BlueAPI client configuration file must be specified in supervisor mode."
+                )
+            if not args.supervisor_config:
+                raise RuntimeError(
+                    "BlueAPI supervisor configuration file must be specified in supervisor mode."
+                )
+
+            client_config = _load_config_from_yaml(Path(args.client_config))
+            supervisor_config = _load_config_from_yaml(Path(args.supervisor_config))
+            context = BlueskyContext(configuration=supervisor_config)
+            plan_runner = SupervisorRunner(context, client_config, args.dev_mode)
+            create_server_for_udc(plan_runner, HyperionConstants.SUPERVISOR_PORT)
+            _register_sigterm_handler(plan_runner)
+            run_forever(plan_runner)
+
+
+def _load_config_from_yaml(config_path: Path):
+    loader = ConfigLoader(ApplicationConfig)
+    loader.use_values_from_yaml(config_path)
+    return loader.load()
 
 
 def _register_sigterm_handler(runner: PlanRunner):

src/mx_bluesky/hyperion/blueapi_config.yaml

@@ -8,10 +8,9 @@ env:
     broadcast_status_events: false
 api:
   url: http://localhost:5005
-  cors:
-    allow_credentials: True
-    origins: 
-      - "*"
+stomp:
+  enabled: true
+  url: tcp://localhost:61613
 logging:
   graylog:
     url: "tcp://graylog-log-target.diamond.ac.uk:12232"

src/mx_bluesky/hyperion/blueapi_dev_config.yaml

@@ -9,9 +9,8 @@ env:
     broadcast_status_events: false
 api:
   url: http://localhost:5005
-  cors:
-    allow_credentials: True
-    origins: 
-      - "*"
+stomp:
+  enabled: true
+  url: tcp://localhost:61613
 logging:
   level: DEBUG

src/mx_bluesky/hyperion/blueapi_plans/__init__.py

@@ -73,13 +73,13 @@ def robot_unload(
 
 def clean_up_udc(
     visit: str,
-    cleanup_group="cleanup",
+    cleanup_group: str = "cleanup",
     robot: BartRobot = inject("robot"),
     smargon: Smargon = inject("smargon"),
     aperture_scatterguard: ApertureScatterguard = inject("aperture_scatterguard"),
     lower_gonio: XYZStage = inject("lower_gonio"),
     detector_motion: DetectorMotion = inject("detector_motion"),
-):
+) -> MsgGenerator:
     yield from bps.abs_set(
         detector_motion.shutter, ShutterState.CLOSED, group=cleanup_group
     )

src/mx_bluesky/hyperion/external_interaction/callbacks/__main__.py

@@ -52,8 +52,8 @@
 from mx_bluesky.hyperion.external_interaction.callbacks.snapshot_callback import (
     BeamDrawingCallback,
 )
-from mx_bluesky.hyperion.parameters.cli import parse_callback_dev_mode_arg
-from mx_bluesky.hyperion.parameters.constants import CONST, HyperionConstants
+from mx_bluesky.hyperion.parameters.cli import CallbackArgs, parse_callback_args
+from mx_bluesky.hyperion.parameters.constants import CONST
 from mx_bluesky.hyperion.parameters.gridscan import (
     GridCommonWithHyperionDetectorParams,
     HyperionSpecifiedThreeDGridScan,
@@ -159,8 +159,8 @@ def wait_for_threads_forever(threads: Sequence[Thread]):
 class HyperionCallbackRunner:
     """Runs Nexus, ISPyB and Zocalo callbacks in their own process."""
 
-    def __init__(self, dev_mode) -> None:
-        setup_logging(dev_mode)
+    def __init__(self, callback_args: CallbackArgs) -> None:
+        setup_logging(callback_args.dev_mode)
         log_info("Hyperion callback process started.")
         set_alerting_service(LoggingAlertService(CONST.GRAYLOG_STREAM_ID))
 
@@ -187,7 +187,12 @@ def start_dispatcher(callbacks: list[Callable]):
             name="0MQ Dispatcher",
         )
 
-        self.watchdog_thread = Thread(target=run_watchdog, daemon=True, name="Watchdog")
+        self.watchdog_thread = Thread(
+            target=run_watchdog,
+            daemon=True,
+            name="Watchdog",
+            args=[callback_args.watchdog_port],
+        )
         log_info("Created 0MQ proxy and local RemoteDispatcher.")
 
     def start(self):
@@ -201,12 +206,12 @@ def start(self):
         )
 
 
-def run_watchdog():
+def run_watchdog(watchdog_port: int):
     log_info("Hyperion watchdog keepalive running")
     while True:
         try:
             with request.urlopen(
-                f"http://localhost:{HyperionConstants.HYPERION_PORT}/callbackPing",
+                f"http://localhost:{watchdog_port}/callbackPing",
                 timeout=PING_TIMEOUT_S,
             ) as response:
                 if response.status != 200:
@@ -219,9 +224,10 @@ def run_watchdog():
 
 
 def main(dev_mode=False) -> None:
-    dev_mode = dev_mode or parse_callback_dev_mode_arg()
+    callback_args = parse_callback_args()
+    callback_args.dev_mode = dev_mode or callback_args.dev_mode
     print(f"In dev mode: {dev_mode}")
-    runner = HyperionCallbackRunner(dev_mode)
+    runner = HyperionCallbackRunner(callback_args)
     runner.start()