Support multiple microgrid controllers and failover

tactical-microgrid-standard/CMakeLists.txt

@@ -57,14 +57,14 @@ add_executable(Controller
   controller/ControllerCommandDataReaderListenerImpl.cpp
   controller/ReplyDataReaderListenerImpl.cpp
   controller/PowerTopologyDataReaderListenerImpl.cpp
+  controller/ActiveMicrogridControllerStateDataReaderListenerImpl.cpp
 )
 target_include_directories(Controller PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_link_libraries(Controller PRIVATE Commands_Idl PowerSim_Idl)
 
 add_executable(CLI
   cli/main.cpp
   cli/CLIClient.cpp
-  cli/ActiveMicrogridControllerStateDataReaderListenerImpl.cpp
 )
 target_include_directories(CLI PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_link_libraries(CLI PRIVATE Commands_Idl PowerSim_Idl)

tactical-microgrid-standard/cli/CLIClient.h

@@ -12,11 +12,6 @@
 #include <mutex>
 #include <unordered_set>
 
-struct UnavailableController {
-  tms::Identity id;
-  static const char* name() { return "UnavailableController"; }
-};
-
 struct ConnectedDeviceEqual {
   bool operator()(const powersim::ConnectedDevice& cd1, const powersim::ConnectedDevice& cd2) const
   {
@@ -31,7 +26,7 @@ struct ConnectedDeviceHash {
   }
 };
 
-class CLIClient : public TimerHandler<UnavailableController> {
+class CLIClient {
 public:
   explicit CLIClient(const tms::Identity& id);
   ~CLIClient() {}
@@ -40,8 +35,6 @@ class CLIClient : public TimerHandler<UnavailableController> {
 
   void run();
 
-  void set_active_controller(const tms::Identity& device_id, const OPENDDS_OPTIONAL_NS::optional<tms::Identity>& master_id);
-
 private:
   // Initialize DDS entities in the TMS domain
   DDS::ReturnCode_t init_tms(DDS::DomainId_t tms_domain_id, int argc = 0, char* argv[] = nullptr);
@@ -57,8 +50,16 @@ class CLIClient : public TimerHandler<UnavailableController> {
 
   void display_commands() const;
   std::string energy_level_to_string(tms::EnergyStartStopLevel essl) const;
+
+  enum class ControllerStatus {
+    AVAILABLE,
+    UNAVAILABLE,
+  };
+
+  ControllerStatus controller_status(TimePoint now, TimePoint last_heartbeat) const;
   void display_controllers() const;
-  void set_controller(const OpArgPair& op_arg);
+  bool collect_power_devices();
+  void display_power_devices() const;
   void list_power_devices();
   bool is_single_port_device(tms::DeviceRole role) const;
 
@@ -69,24 +70,23 @@ class CLIClient : public TimerHandler<UnavailableController> {
   // Create a power connection between two devices
   void connect(const tms::Identity& id1, tms::DeviceRole role1,
     const tms::Identity& id2, tms::DeviceRole role2);
-
+  void consolidate_power_devices();
   void connect_power_devices();
-  bool send_power_devices_request();
-  void display_power_devices() const;
-  void send_start_device_cmd(const OpArgPair& op_arg) const;
-  void send_stop_device_cmd(const OpArgPair& op_arg) const;
-  void send_start_stop_request(const OpArgPair& op_arg, tms::OperatorPriorityType opt) const;
-  void send_stop_controller_cmd() const;
-  void send_resume_controller_cmd() const;
-  void send_terminate_controller_cmd() const;
-  void send_controller_cmd(cli::ControllerCmdType cmd_type) const;
+  bool send_power_devices_request(const tms::Identity& mc_id);
+  void send_start_device_cmd(const OpArgPair& op_arg);
+  void send_stop_device_cmd(const OpArgPair& op_arg);
+  void send_start_stop_request(const OpArgPair& op_arg, tms::OperatorPriorityType opt);
+  void send_suspend_controller_cmd(const OpArgPair& op_pair) const;
+  void send_resume_controller_cmd(const OpArgPair& op_pair) const;
+  void send_terminate_controller_cmd(const OpArgPair& op_pair) const;
+  void send_controller_cmd(const OpArgPair& op_pair, cli::ControllerCmdType cmd_type) const;
 
   void process_device_info(const tms::DeviceInfo& di, const DDS::SampleInfo& si);
   void process_heartbeat(const tms::Heartbeat& hb, const DDS::SampleInfo& si);
 
-  void any_timer_fired(TimerHandler<UnavailableController>::AnyTimer any_timer) final;
   int handle_signal(int, siginfo_t*, ucontext_t*);
 
+  ACE_Reactor* const reactor_ = ACE_Reactor::instance();
   DDS::DomainParticipant_var sim_participant_;
   Handshaking handshaking_;
   cli::PowerDevicesRequestDataWriter_var pdreq_dw_;
@@ -104,39 +104,30 @@ class CLIClient : public TimerHandler<UnavailableController> {
   static constexpr Sec unavail_controller_delay = missed_controller_delay + lost_controller_delay;
 
   struct ControllerInfo {
-    enum class Status {
-      AVAILABLE,
-      UNAVAILABLE,
-    };
-
     tms::DeviceInfo info;
-    Status status;
     TimePoint last_hb;
   };
 
   mutable std::mutex cli_m_;
   bool stop_cli_;
 
+  // Mutex for the following data
   mutable std::mutex data_m_;
 
-  // Microgrid controllers to which the CLI client is connected
+  // Microgrid controllers that are (and were) reachable by the CLI
   std::unordered_map<tms::Identity, ControllerInfo> controllers_;
 
-  // The power devices that are connected to the current controller
+  // For each controller, cache the power devices that select it as the active controller.
+  // This mapping can change during the operation of the microgrid due to the availability of the MCs.
+  std::unordered_map<tms::Identity, PowerDevices> mc_to_devices_;
+
+  // All power devices in the microgrid, reported by the MCs.
   PowerDevices power_devices_;
 
   // Store the simulated power connections between power devices
-  using PowerConnection = std::unordered_map<tms::Identity, std::unordered_set<powersim::ConnectedDevice, ConnectedDeviceHash, ConnectedDeviceEqual>>;
+  using PowerConnection = std::unordered_map<tms::Identity,
+    std::unordered_set<powersim::ConnectedDevice, ConnectedDeviceHash, ConnectedDeviceEqual>>;
   PowerConnection power_connections_;
-
-  // The current microgrid controller with which the CLI client is interacting
-  tms::Identity curr_controller_;
-
-  mutable std::mutex active_controllers_m_;
-
-  // Active controller selected by each power device (power device => its controller).
-  // Can be used to check that all power devices will eventually select the same active controller.
-  std::map<tms::Identity, tms::Identity> active_controllers_;
 };
 
 #endif

tactical-microgrid-standard/cli_idl/CLICommands.idl

@@ -13,10 +13,13 @@ module cli {
   };
 
   @nested
-  @extensibility(FINAL)
+  @extensibility(APPENDABLE)
   struct PowerDeviceInfo {
     tms::DeviceInfo device_info;
     tms::EnergyStartStopLevel essl;
+    // The MC that controls this device.
+    // Obtain from tms::ActiveMicrogridControllerState sent by the device.
+    @optional tms::Identity master_id;
   };
 
   typedef sequence<PowerDeviceInfo> PowerDeviceInfoSeq;

tactical-microgrid-standard/common/ControllerSelector.cpp

@@ -50,24 +50,20 @@ void ControllerSelector::timer_fired(Timer<NewController>& timer)
   Guard g(lock_);
   const auto& mc_id = timer.arg.id;
   ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(NewController): "
-    "\"%C\" -> \"%C\"\n", selected_.c_str(), mc_id.c_str()));
+    "triggered by \"%C\" heartbeat\n", mc_id.c_str()));
 
-  // The TMS spec isn't clear to whether the device needs to verify that the last
-  // heartbeat of this controller was received less than 3s (i.e., heartbeat deadline) ago.
-  // This check makes sense since if its last heartbeat was more than 3s ago, that means
-  // the controller is not available and should not be selected as the active controller.
-  const TimePoint now = Clock::now();
-  auto it = all_controllers_.find(mc_id);
-  if (it == all_controllers_.end()) {
+  if (all_controllers_.find(mc_id) == all_controllers_.end()) {
     ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: ControllerSelector::timed_event(NewController): Controller \"%C\" not found!\n",
                mc_id.c_str()));
     return;
   }
 
-  if (now - it->second < heartbeat_deadline) {
-    selected_ = mc_id;
-    send_controller_state();
-  }
+  // Select the highest priority controller from the list of available controllers.
+  // The TMS spec doesn't specify this, and an implementation may select the controller
+  // associated with this timer. However, that could cause different controllers to be
+  // selected by different devices, since each device may receive the first heartbeat from
+  // a different controller and thus select a different controller when the timer expires.
+  select_controller();
 }
 
 void ControllerSelector::timer_fired(Timer<MissedHeartbeat>& timer)
@@ -102,14 +98,7 @@ void ControllerSelector::timer_fired(Timer<LostController>&)
 
   // Select a new controller if possible. If there are no recent controllers
   // and we don't hear from another, the NoControllers timer will fire.
-  const TimePoint now = Clock::now();
-  for (auto it = all_controllers_.begin(); it != all_controllers_.end(); ++it) {
-    const auto last_hb = now - it->second;
-    if (last_hb < heartbeat_deadline) {
-      select(it->first, std::chrono::duration_cast<Sec>(last_hb));
-      break;
-    }
-  }
+  select_controller();
 }
 
 void ControllerSelector::timer_fired(Timer<NoControllers>&)
@@ -119,6 +108,25 @@ void ControllerSelector::timer_fired(Timer<NoControllers>&)
   // TODO: CONFIG_ON_COMMS_LOSS
 }
 
+// Select a new controller at start up,
+// or when a device loses its active controller and has to select a new one.
+bool ControllerSelector::select_controller()
+{
+  const TimePoint now = Clock::now();
+
+  // Select an available controller with smallest identity alphabetically
+  for (auto it = all_controllers_.begin(); it != all_controllers_.end(); ++it) {
+    const auto last_hb = now - it->second;
+    // TMS spec doesn't specify this. But it should make sure the controller is still available
+    // i.e., last heartbeat received within 3 seconds.
+    if (last_hb < heartbeat_deadline) {
+      select(it->first, std::chrono::duration_cast<Sec>(last_hb));
+      return true;
+    }
+  }
+  return false;
+}
+
 void ControllerSelector::select(const tms::Identity& id, Sec last_hb)
 {
   ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::select: \"%C\"\n", id.c_str()));

tactical-microgrid-standard/common/ControllerSelector.h

@@ -103,6 +103,7 @@ class OpenDDS_TMS_Export ControllerSelector :
 
   void select(const tms::Identity& id, Sec last_hb = Sec(0));
 
+  bool select_controller();
   void send_controller_state();
 
   tms::Identity selected_;

tactical-microgrid-standard/cli/ActiveMicrogridControllerStateDataReaderListenerImpl.cpp → tactical-microgrid-standard/controller/ActiveMicrogridControllerStateDataReaderListenerImpl.cpp

@@ -17,7 +17,7 @@ void ActiveMicrogridControllerStateDataReaderListenerImpl::on_data_available(DDS
     if (info_seq[i].valid_data) {
       const tms::Identity& device_id = data[i].deviceId();
       auto master_id = data[i].masterId();
-      cli_client_.set_active_controller(device_id, master_id);
+      controller_.set_active_controller(device_id, master_id);
     }
   }
 }

tactical-microgrid-standard/cli/ActiveMicrogridControllerStateDataReaderListenerImpl.h → tactical-microgrid-standard/controller/ActiveMicrogridControllerStateDataReaderListenerImpl.h

@@ -1,21 +1,21 @@
-#ifndef CLI_ACTIVE_MICROGRID_CONTROLLER_STATE_DATA_READER_LISTENER_IMPL_H
-#define CLI_ACTIVE_MICROGRID_CONTROLLER_STATE_DATA_READER_LISTENER_IMPL_H
+#ifndef ACTIVE_MICROGRID_CONTROLLER_STATE_DATA_READER_LISTENER_IMPL_H
+#define ACTIVE_MICROGRID_CONTROLLER_STATE_DATA_READER_LISTENER_IMPL_H
 
 #include "common/DataReaderListenerBase.h"
-#include "CLIClient.h"
+#include "Controller.h"
 
 class ActiveMicrogridControllerStateDataReaderListenerImpl : public DataReaderListenerBase {
 public:
-  explicit ActiveMicrogridControllerStateDataReaderListenerImpl(CLIClient& cli_client)
+  explicit ActiveMicrogridControllerStateDataReaderListenerImpl(Controller& controller)
     : DataReaderListenerBase("tms::ActiveMicrogridControllerState - DataReaderListenerImpl")
-    , cli_client_(cli_client) {}
+    , controller_(controller) {}
 
   virtual ~ActiveMicrogridControllerStateDataReaderListenerImpl() = default;
 
   void on_data_available(DDS::DataReader_ptr reader) final;
 
 private:
-  CLIClient& cli_client_;
+  Controller& controller_;
 };
 
 #endif

tactical-microgrid-standard/controller/CLIServer.cpp

@@ -384,6 +384,14 @@ void CLIServer::start_stop_device(const tms::Identity& pd_id, tms::OperatorPrior
     return;
   }
 
+  // Only update the energy level of the device locally if this is not its active controller.
+  const auto& master_id = it->second.master_id();
+  if (!master_id.has_value() || master_id.value() != controller_.get_device_id()) {
+    controller_.update_essl(pd_id, to_essl);
+    return;
+  }
+
+  // Else, send a command to the device to actually start/stop it.
   tms::EnergyStartStopRequest essr;
   essr.requestId().requestingDeviceId(controller_.id());
   essr.requestId().targetDeviceId(pd_id);

tactical-microgrid-standard/controller/Common.h

@@ -4,11 +4,10 @@
 #include <common/mil-std-3071_data_modelTypeSupportImpl.h>
 #include <cli_idl/CLICommandsTypeSupportImpl.h>
 
-#include <dds/DCPS/optional.h>
-
 #include <unordered_map>
+#include <optional>
 
-using OpArgPair = std::pair<std::string, OpenDDS::DCPS::optional<std::string>>;
+using OpArgPair = std::pair<std::string, std::optional<std::string>>;
 using PowerDevices = std::unordered_map<tms::Identity, cli::PowerDeviceInfo>;
 
 #endif