From d223cda8677a2b62b72b4f5799d57bf278aeffd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Fiti=C3=A9?= Date: Wed, 27 Aug 2025 11:23:16 +0200 Subject: [PATCH 1/2] TARDA-2840 Refactor DBMDataRef for thread-safe annotation access and memory-efficient size limiting (#327) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Thread-safe access to _annotations in DBMDataRef using lock Signed-off-by: Johan Fitié * Limit _annotations dictionary size to 10,000 by removing entries when exceeded Signed-off-by: Johan Fitié * Refactor to use constant for maximum annotations size Signed-off-by: Johan Fitié * Refactor _annotations to use SortedDictionary for FIFO eviction instead of random removal Signed-off-by: Johan Fitié * Limit line length correctly Signed-off-by: Johan Fitié * Update copyright year, change maximum number of annotations Signed-off-by: Johan Fitié --------- Signed-off-by: Johan Fitié --- README.md | 2 +- src/DBMDataRef/DBMDataRef.vb | 60 +++++++++++++++++++++++++----------- src/dbm/DBMManifest.vb | 4 +-- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 6effc52c..6099943b 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # DBM Dynamic Bandwidth Monitor\ Leak detection method implemented in a real-time data historian\ -Copyright (C) 2014-2024 J.H. Fitié, Vitens N.V. +Copyright (C) 2014-2025 J.H. Fitié, Vitens N.V. ## Description Water company Vitens has created a demonstration site called the Vitens Innovation Playground (VIP), in which new technologies and methodologies are developed, tested, and demonstrated. The projects conducted in the demonstration site can be categorized into one of four themes: energy optimization, real-time leak detection, online water quality monitoring, and customer interaction. In the real-time leak detection theme, a method for leak detection based on statistical demand forecasting was developed. diff --git a/src/DBMDataRef/DBMDataRef.vb b/src/DBMDataRef/DBMDataRef.vb index 19c01d79..d0751d91 100644 --- a/src/DBMDataRef/DBMDataRef.vb +++ b/src/DBMDataRef/DBMDataRef.vb @@ -1,6 +1,6 @@ ' Dynamic Bandwidth Monitor ' Leak detection method implemented in a real-time data historian -' Copyright (C) 2014-2024 J.H. Fitié, Vitens N.V. +' Copyright (C) 2014-2025 J.H. Fitié, Vitens N.V. ' ' This file is part of DBM. ' @@ -25,6 +25,7 @@ Imports System.DateTime Imports System.Double Imports System.Math Imports System.Runtime.InteropServices +Imports System.Threading Imports OSIsoft.AF.Asset Imports OSIsoft.AF.Asset.AFAttributeTrait Imports OSIsoft.AF.Data @@ -83,13 +84,15 @@ Namespace Vitens.DynamicBandwidthMonitor ' process output. + Const MaxAnnotationsSize As Integer = 1000 ' Maximum number of annotations Const StaleDataMinutes As Integer = 10 ' Minutes until snapshot is stale Const CategoryNoCorrelation As String = "NoCorrelation" Const pValueLoHi As Double = 0.95 ' Confidence interval for Lo and Hi Const pValueMinMax As Double = 0.9999 ' CI for Minimum and Maximum - Private _annotations As New Dictionary(Of AFTime, Object) + Private _lock As New Object ' Object for exclusive lock on critical section. + Private _annotations As New SortedDictionary(Of AFTime, Object) Private Shared _dbm As New DBM(New DBMLoggerAFTrace) @@ -279,20 +282,31 @@ Namespace Vitens.DynamicBandwidthMonitor If value IsNot Nothing Then ' Key - _annotations.Remove(value.Timestamp) ' Remove existing - value.Annotated = False + Monitor.Enter(_lock) ' Block + Try - If annotation IsNot Nothing Then ' Value + _annotations.Remove(value.Timestamp) ' Remove existing + value.Annotated = False - DBM.Logger.LogDebug( - "value.Timestamp " & - value.Timestamp.LocalTime.ToString("s") & "; " & - "annotation " & DirectCast(annotation, String), Attribute.GetPath) + If annotation IsNot Nothing Then ' Value - _annotations.Add(value.Timestamp, annotation) ' Add - value.Annotated = True + While _annotations.Count >= MaxAnnotationsSize ' Limit size + _annotations.Remove(_annotations.Keys.First()) ' Remove oldest + End While - End If + DBM.Logger.LogDebug( + "value.Timestamp " & + value.Timestamp.LocalTime.ToString("s") & "; " & + "annotation " & DirectCast(annotation, String), Attribute.GetPath) + + _annotations.Add(value.Timestamp, annotation) ' Add + value.Annotated = True + + End If + + Finally + Monitor.Exit(_lock) ' Unblock + End Try End If @@ -308,12 +322,22 @@ Namespace Vitens.DynamicBandwidthMonitor ' attributes" GetAnnotation = Nothing - If value IsNot Nothing AndAlso - _annotations.TryGetValue(value.Timestamp, GetAnnotation) Then - _annotations.Remove(value.Timestamp) ' Remove after get - Return GetAnnotation - Else - Return String.Empty ' Default + If value IsNot Nothing Then ' Key + + Monitor.Enter(_lock) ' Block + Try + + If _annotations.TryGetValue(value.Timestamp, GetAnnotation) Then + _annotations.Remove(value.Timestamp) ' Remove after get + Return GetAnnotation + Else + Return String.Empty ' Default + End If + + Finally + Monitor.Exit(_lock) ' Unblock + End Try + End If End Function diff --git a/src/dbm/DBMManifest.vb b/src/dbm/DBMManifest.vb index 25bb2939..e0223bae 100644 --- a/src/dbm/DBMManifest.vb +++ b/src/dbm/DBMManifest.vb @@ -1,6 +1,6 @@ ' Dynamic Bandwidth Monitor ' Leak detection method implemented in a real-time data historian -' Copyright (C) 2014-2024 J.H. Fitié, Vitens N.V. +' Copyright (C) 2014-2025 J.H. Fitié, Vitens N.V. ' ' This file is part of DBM. ' @@ -30,6 +30,6 @@ "Leak detection method implemented in a real-time data historian")> + "Copyright (C) 2014-2025 J.H. Fitié, Vitens N.V.")> From 905d12066a7d2dc837df434992a72c4e9b0538a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Fiti=C3=A9?= Date: Fri, 29 Aug 2025 09:23:52 +0200 Subject: [PATCH 2/2] TARDA-2840 Make DBM instance non-shared in DBMDataRef and cap DBMDataStore size with pruning (#330) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Fitié --- src/DBMDataRef/DBMDataRef.vb | 2 +- src/dbm/DBMDataStore.vb | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/DBMDataRef/DBMDataRef.vb b/src/DBMDataRef/DBMDataRef.vb index d0751d91..3cb8aa5f 100644 --- a/src/DBMDataRef/DBMDataRef.vb +++ b/src/DBMDataRef/DBMDataRef.vb @@ -93,7 +93,7 @@ Namespace Vitens.DynamicBandwidthMonitor Private _lock As New Object ' Object for exclusive lock on critical section. Private _annotations As New SortedDictionary(Of AFTime, Object) - Private Shared _dbm As New DBM(New DBMLoggerAFTrace) + Private _dbm As New DBM(New DBMLoggerAFTrace) Public Shared Function CreateDataPipe As Object diff --git a/src/dbm/DBMDataStore.vb b/src/dbm/DBMDataStore.vb index e5c985c3..c3adcdbf 100644 --- a/src/dbm/DBMDataStore.vb +++ b/src/dbm/DBMDataStore.vb @@ -1,6 +1,6 @@ ' Dynamic Bandwidth Monitor ' Leak detection method implemented in a real-time data historian -' Copyright (C) 2014-2022 J.H. Fitié, Vitens N.V. +' Copyright (C) 2014-2025 J.H. Fitié, Vitens N.V. ' ' This file is part of DBM. ' @@ -22,6 +22,7 @@ Imports System Imports System.Collections.Generic Imports System.Double Imports System.Threading +Imports Vitens.DynamicBandwidthMonitor.DBMParameters Namespace Vitens.DynamicBandwidthMonitor @@ -30,8 +31,12 @@ Namespace Vitens.DynamicBandwidthMonitor Public Class DBMDataStore + Private Shared ReadOnly MaxDataStoreSize As Integer = + (365+ComparePatterns*7)*24*60*60\CalculationInterval+EMAPreviousPeriods + + Private _lock As New Object ' Object for exclusive lock on critical section. - Private _dataStore As New Dictionary(Of DateTime, Double) ' In-memory data + Private _dataStore As New SortedDictionary(Of DateTime, Double) ' In-mem Public Sub AddData(timestamp As DateTime, data As Object) @@ -46,7 +51,13 @@ Namespace Vitens.DynamicBandwidthMonitor Try If Not _dataStore.ContainsKey(timestamp) Then + + While _dataStore.Count >= MaxDataStoreSize ' Limit size + _dataStore.Remove(_dataStore.Keys.First()) ' Remove oldest + End While + _dataStore.Add(timestamp, DirectCast(data, Double)) + End If Finally