Mqtt integration

.github/workflows/test-mqtt-broker.yml

@@ -0,0 +1,55 @@
+name: Test BELY MQTT Message Broker
+
+on:
+  pull_request:
+    paths:
+      - 'tools/developer_tools/bely-mqtt-message-broker/**'
+      - '.github/workflows/test-mqtt-broker.yml'
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Run MQTT Broker Tests
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: tools/developer_tools/bely-mqtt-message-broker
+
+    strategy:
+      matrix:
+        python-version: ['3.11', '3.12']
+        # python-version: ['3.11']
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          make install-dev      
+
+      - name: Run code quality linter
+        run: |
+          make lint
+
+      - name: Run code quality type-check
+        run: |
+          make type-check
+
+      - name: Run tests with coverage
+        run: |
+          make test-cov
+
+      # - name: Upload coverage reports
+      #   if: matrix.python-version == '3.11'
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     name: coverage-report
+      #     path: tools/developer_tools/bely-mqtt-message-broker/htmlcov/

.gitignore

@@ -2,6 +2,7 @@
 .vscode
 .DS_Store
 src/python/.loglogin
+.loglogin
 
 # Compiled files 
 *.pyc
@@ -56,3 +57,5 @@ docs/python/_build/
 # Temporary testing csv files
 tools/developer_tools/python-client/cdbCli/service/cli/Spreadsheets/
 .env
+tools/developer_tools/bely-mqtt-message-broker/conda-bld
+tools/developer_tools/bely-mqtt-message-broker/dev-config

etc/bely-mqtt-template

@@ -0,0 +1,157 @@
+keys:
+    - name: topic
+      type: string
+      color:
+        foreground: crimson
+        background: black
+      max-width: 25
+    - name: description
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: entityId
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: entityName
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: eventTimestamp
+      type: string
+      color:
+        foreground: limegreen
+        background: black
+      max-width: 25
+    - name: eventTriggedByUsername
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: logInfo/enteredByUsername
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: logInfo/enteredOnDateTime
+      type: string
+      color:
+        foreground: limegreen
+        background: black
+      max-width: 25
+    - name: logInfo/id
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: logInfo/lastModifiedByUsername
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: logInfo/lastModifiedOnDateTime
+      type: string
+      color:
+        foreground: limegreen
+        background: black
+      max-width: 25
+    - name: logbookList
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: parentLogDocumentInfo/createdByUsername
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: parentLogDocumentInfo/enteredOnDateTime
+      type: string
+      color:
+        foreground: limegreen
+        background: black
+      max-width: 25
+    - name: parentLogDocumentInfo/id
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: parentLogDocumentInfo/lastModifiedByUsername
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: parentLogDocumentInfo/lastModifiedOnDateTime
+      type: string
+      color:
+        foreground: limegreen
+        background: black
+      max-width: 25
+    - name: parentLogDocumentInfo/name
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: parentLogDocumentInfo/ownerUserGroupName
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: parentLogDocumentInfo/ownerUsername
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: parentlogInfo/enteredByUsername
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: parentlogInfo/enteredOnDateTime
+      type: string
+      color:
+        foreground: limegreen
+        background: black
+      max-width: 25
+    - name: parentlogInfo/id
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: parentlogInfo/lastModifiedByUsername
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25
+    - name: parentlogInfo/lastModifiedOnDateTime
+      type: string
+      color:
+        foreground: limegreen
+        background: black
+      max-width: 25
+    - name: textDiff
+      type: string
+      color:
+        foreground: white
+        background: black
+      max-width: 25

sbin/bely_configure_mqtt_service.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+
+# Copyright (c) UChicago Argonne, LLC. All rights reserved.
+# See LICENSE file.
+
+
+#
+# Script used for configuring MQTT connector for Bely webapp
+# Deploys MQTT Resource Adapter and creates connection pool/resource
+#
+# Usage:
+#
+# $0 [mqtt_config_file]
+#
+# If no config file is specified, defaults to $LOGR_INSTALL_DIR/etc/mqtt.conf
+#
+# Sample MQTT configuration file contents (mqtt.conf):
+# MQTT_HOST=localhost              # MQTT broker hostname (default: localhost)
+# MQTT_PORT=1883                   # MQTT broker port (default: 1883)
+# MQTT_USERNAME=admin              # MQTT username (optional)
+# MQTT_PASSWORD=admin              # MQTT password (optional)
+# MQTT_CLEAN_SESSION=true          # Clean session flag (optional)
+# MQTT_QOS=1                       # Quality of Service level (optional)
+# MQTT_KEEP_ALIVE_INTERVAL=60      # Keep alive interval in seconds (optional)
+# MQTT_CONNECTION_TIMEOUT=30       # Connection timeout in seconds (optional)
+# MQTT_MAX_INFLIGHT=10             # Maximum number of messages in flight (optional)
+# MQTT_AUTOMATIC_RECONNECT=true    # Automatic reconnection on disconnect (optional)
+# MQTT_FILE_PERSISTANCE=false      # Enable file-based message persistence (optional)
+# MQTT_PERSISTENCE_DIRECTORY=.     # Directory for persistent message storage (optional)
+# MQTT_TOPIC_FILTER                # MQTT topic filter for subscriptions (optional)
+
+MY_DIR=`dirname $0` && cd $MY_DIR && MY_DIR=`pwd`
+if [ -z "${BELY_ROOT_DIR}" ]; then
+    BELY_ROOT_DIR=$MY_DIR/..
+fi
+BELY_ENV_FILE=${BELY_ROOT_DIR}/setup.sh
+if [ ! -f ${BELY_ENV_FILE} ]; then
+    echo "Environment file ${BELY_ENV_FILE} does not exist."
+    exit 2
+fi
+. ${BELY_ENV_FILE} > /dev/null
+
+# Constants
+MQTT_POOL_NAME="bely/MQTT/pool"
+MQTT_RESOURCE_NAME="bely/MQTT/resource"
+MQTT_RAR_NAME="mqtt-rar-0.8.0"
+MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME="mqtt-rar-deployment"
+MQTT_RAR_PATH=$BELY_ROOT_DIR/src/lib/${MQTT_RAR_NAME}.rar
+
+# Look for MQTT configuration file
+if [ ! -z "$1" ]; then
+    mqttConfigFile=$1
+else
+    mqttConfigFile=$LOGR_INSTALL_DIR/etc/mqtt.conf
+fi
+
+if [ -f $mqttConfigFile ]; then
+    echo "Using MQTT config file: $mqttConfigFile"
+    . $mqttConfigFile
+else
+    echo "Error: MQTT config file $mqttConfigFile not found."
+    echo "You can create one using bely_create_mqtt_configuration.sh"
+    exit 1
+fi
+
+BELY_HOST_ARCH=$(uname -sm | tr -s '[:upper:][:blank:]' '[:lower:][\-]')
+GLASSFISH_DIR=$LOGR_SUPPORT_DIR/payara/$BELY_HOST_ARCH
+
+ASADMIN_CMD=$GLASSFISH_DIR/bin/asadmin
+
+# MQTT Configuration defaults
+MQTT_HOST=${MQTT_HOST:=localhost}
+MQTT_PORT=${MQTT_PORT:=1883}
+
+# Build properties string for connection pool
+PROPERTIES="serverURIs=tcp\\://${MQTT_HOST}\\:${MQTT_PORT}"
+
+if [ ! -z "$MQTT_USERNAME" ]; then
+    PROPERTIES="${PROPERTIES}:userName=${MQTT_USERNAME}"
+fi
+
+if [ ! -z "$MQTT_PASSWORD" ]; then
+    PROPERTIES="${PROPERTIES}:password=${MQTT_PASSWORD}"
+fi
+
+if [ ! -z "$MQTT_CLEAN_SESSION" ]; then
+    PROPERTIES="${PROPERTIES}:cleanSession=${MQTT_CLEAN_SESSION}"
+fi
+
+if [ ! -z "$MQTT_KEEP_ALIVE_INTERVAL" ]; then
+    PROPERTIES="${PROPERTIES}:keepAliveInterval=${MQTT_KEEP_ALIVE_INTERVAL}"
+fi
+
+if [ ! -z "$MQTT_CONNECTION_TIMEOUT" ]; then
+    PROPERTIES="${PROPERTIES}:connectionTimeout=${MQTT_CONNECTION_TIMEOUT}"
+fi
+
+if [ ! -z "$MQTT_MAX_INFLIGHT" ]; then
+    PROPERTIES="${PROPERTIES}:maxInflight=${MQTT_MAX_INFLIGHT}"
+fi
+
+if [ ! -z "$MQTT_AUTOMATIC_RECONNECT" ]; then
+    PROPERTIES="${PROPERTIES}:automaticReconnect=${MQTT_AUTOMATIC_RECONNECT}"
+fi
+
+if [ ! -z "$MQTT_FILE_PERSISTANCE" ]; then
+    PROPERTIES="${PROPERTIES}:filePersistance=${MQTT_FILE_PERSISTANCE}"
+fi
+
+if [ ! -z "$MQTT_PERSISTENCE_DIRECTORY" ]; then
+    PROPERTIES="${PROPERTIES}:persistenceDirectory=${MQTT_PERSISTENCE_DIRECTORY}"
+fi
+
+if [ ! -z "$MQTT_QOS" ]; then
+    PROPERTIES="${PROPERTIES}:qos=${MQTT_QOS}"
+fi
+
+if [ ! -z "$MQTT_TOPIC_FILTER" ]; then
+    PROPERTIES="${PROPERTIES}:topicFilter=${MQTT_TOPIC_FILTER}"
+fi
+
+# Deploy MQTT RAR
+echo "Deploying MQTT RAR"
+if [ -f "$MQTT_RAR_PATH" ]; then
+    # Check if already deployed and undeploy if needed
+    $ASADMIN_CMD list-applications | grep -q ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} && {
+        echo "Undeploying existing MQTT RAR"
+        # Check if resource exists and delete it
+        $ASADMIN_CMD list-connector-resources | grep -q ${MQTT_RESOURCE_NAME} && {
+            echo "Deleting existing MQTT resource"
+            $ASADMIN_CMD delete-connector-resource ${MQTT_RESOURCE_NAME} || exit 1
+        }
+
+        # Check if connection pool exists and delete it
+        $ASADMIN_CMD list-connector-connection-pools | grep -q ${MQTT_POOL_NAME} && {
+            echo "Deleting existing MQTT connection pool"
+            $ASADMIN_CMD delete-connector-connection-pool ${MQTT_POOL_NAME} || exit 1
+        }
+
+        echo "Undeploying existing MQTT RAR"
+        $ASADMIN_CMD undeploy ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} || exit 1
+    }
+    $ASADMIN_CMD deploy --name ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} $MQTT_RAR_PATH || exit 1
+else
+    echo "Warning: MQTT RAR file not found at $MQTT_RAR_PATH"
+    exit 1
+fi
+
+# Create MQTT connection pool
+echo "Creating MQTT connection pool ${MQTT_POOL_NAME}"
+$ASADMIN_CMD create-connector-connection-pool \
+    --raname ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} \
+    --connectiondefinition fish.payara.cloud.connectors.mqtt.api.MQTTConnectionFactory \
+    --property "${PROPERTIES}" \
+    ${MQTT_POOL_NAME} || exit 1
+# Create MQTT resource
+echo "Creating MQTT resource ${MQTT_RESOURCE_NAME}"
+$ASADMIN_CMD create-connector-resource \
+    --poolname ${MQTT_POOL_NAME} \
+    ${MQTT_RESOURCE_NAME} || exit 1
+# Test MQTT connection pool
+echo "Testing MQTT connection pool"
+$ASADMIN_CMD ping-connection-pool ${MQTT_POOL_NAME} || { echo "Warning: MQTT connection pool ping failed"; exit 1; }
+
+echo "Restart or redeploy BELY."