From 9e8a043917f6cfea3a57154b9ac552425d63d0b8 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Tue, 1 Sep 2015 09:03:31 +0100 Subject: [PATCH 01/30] Added option to periodically check for new instances --- .../MetricsConfiguration.cs | 8 ++++ .../WindowsPerformanceCounterPlugin.cs | 41 +++++++++++++++---- src/CollectdWinService/app.config | 2 +- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/CollectdWinService/MetricsConfiguration.cs b/src/CollectdWinService/MetricsConfiguration.cs index 65d57bf..f8c6f34 100644 --- a/src/CollectdWinService/MetricsConfiguration.cs +++ b/src/CollectdWinService/MetricsConfiguration.cs @@ -550,6 +550,14 @@ public CounterConfigCollection Counters get { return (CounterConfigCollection) base["Counters"]; } set { base["Counters"] = value; } } + + + [ConfigurationProperty("ReloadInterval", IsRequired =false, DefaultValue = 3600)] + public int ReloadInterval + { + get { return (int)base["ReloadInterval"]; } + set { base["ReloadInterval"] = value; } + } } } } diff --git a/src/CollectdWinService/WindowsPerformanceCounterPlugin.cs b/src/CollectdWinService/WindowsPerformanceCounterPlugin.cs index ef198f5..766ac4b 100644 --- a/src/CollectdWinService/WindowsPerformanceCounterPlugin.cs +++ b/src/CollectdWinService/WindowsPerformanceCounterPlugin.cs @@ -24,6 +24,8 @@ internal class WindowsPerformanceCounterPlugin : ICollectdReadPlugin private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); private readonly IList _metrics; private string _hostName; + private int _reloadInterval; + private double _lastUpdated; public WindowsPerformanceCounterPlugin() { @@ -40,19 +42,29 @@ public void Configure() _hostName = Util.GetHostName(); - _metrics.Clear(); + // Set reload time + _reloadInterval = config.WindowsPerformanceCounters.ReloadInterval; + Logger.Info("Loading metric configuration. Reload interval: {0} sec", _reloadInterval); + + _lastUpdated = Util.GetNow(); + // Load the metrics - this checks for existence + _metrics.Clear(); + int metricCounter = 0; foreach (CollectdWinConfig.CounterConfig counter in config.WindowsPerformanceCounters.Counters) { if (counter.Instance == "") { // Instance not specified - AddPerformanceCounter(counter.Category, counter.Name, + if (AddPerformanceCounter(counter.Category, counter.Name, counter.Instance, counter.Multiplier, counter.DecimalPlaces, counter.CollectdPlugin, counter.CollectdPluginInstance, counter.CollectdType, - counter.CollectdTypeInstance); + counter.CollectdTypeInstance)) + { + metricCounter++; + } } else { @@ -112,15 +124,18 @@ public void Configure() } // Replace collectd_plugin_instance with the Instance got from counter - AddPerformanceCounter(counter.Category, counter.Name, + if (AddPerformanceCounter(counter.Category, counter.Name, instance, counter.Multiplier, counter.DecimalPlaces, counter.CollectdPlugin, instanceAlias, counter.CollectdType, - counter.CollectdTypeInstance); + counter.CollectdTypeInstance)) + { + metricCounter++; + } } } } - Logger.Info("WindowsPerformanceCounter plugin configured"); + Logger.Info("WindowsPerformanceCounter plugin configured {0} metrics", metricCounter); } public void Start() @@ -135,6 +150,13 @@ public void Stop() public IList Read() { + // Check if it is time to reload the metric config + // We need to periodially reload the config in case new instances or new categories are available (e.g. due to cluster move) + if (Util.GetNow() - _lastUpdated >= _reloadInterval) + { + Configure(); + } + var metricValueList = new List(); foreach (Metric metric in _metrics) { @@ -177,7 +199,7 @@ public IList Read() return (metricValueList); } - private void AddPerformanceCounter(string category, string names, string instance, double multiplier, + private Boolean AddPerformanceCounter(string category, string names, string instance, double multiplier, int decimalPlaces, string collectdPlugin, string collectdPluginInstance, string collectdType, string collectdTypeInstance) { @@ -213,7 +235,8 @@ private void AddPerformanceCounter(string category, string names, string instanc metric.CollectdType = collectdType; metric.CollectdTypeInstance = collectdTypeInstance; _metrics.Add(metric); - Logger.Info("Added Performance counter : {0}", logstr); + Logger.Debug("Added Performance counter : {0}", logstr); + return true; } catch (InvalidOperationException ex) { @@ -221,10 +244,12 @@ private void AddPerformanceCounter(string category, string names, string instanc Logger.Warn(string.Format("Performance Counter not added: Category does not exist: {0}", category)); } else Logger.Error(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", category, instance, names), ex); + return false; } catch (Exception ex) { Logger.Error(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", category, instance, names), ex); + return false; } } } diff --git a/src/CollectdWinService/app.config b/src/CollectdWinService/app.config index 43aeed2..9af17dd 100644 --- a/src/CollectdWinService/app.config +++ b/src/CollectdWinService/app.config @@ -47,7 +47,7 @@ - + Date: Wed, 24 Feb 2016 10:49:28 +0000 Subject: [PATCH 02/30] PHX-3831 - don't send empty payload when no events --- src/CollectdWinService/WriteNetuitivePlugin.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CollectdWinService/WriteNetuitivePlugin.cs b/src/CollectdWinService/WriteNetuitivePlugin.cs index b9f81b3..ce9877b 100644 --- a/src/CollectdWinService/WriteNetuitivePlugin.cs +++ b/src/CollectdWinService/WriteNetuitivePlugin.cs @@ -99,7 +99,8 @@ public void Write(Queue values) List eventList = ConvertEventsToIngestEvents(events); // Send event payloads - PostEvents(eventList); + if (eventList.Count > 0) + PostEvents(eventList); } From b0476d7fd1a03e8a8ef365984b9ca901a09bc41b Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Wed, 24 Feb 2016 10:50:35 +0000 Subject: [PATCH 03/30] PHX-3836. Add IP address to common attributes --- .../ReadWindowsAttributesPlugin.cs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs index b9ea4dd..0bd04c7 100644 --- a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs +++ b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs @@ -11,6 +11,7 @@ using System.Runtime.Serialization; using System.Runtime.Serialization.Json; using System.IO; +using System.Net.Sockets; namespace Netuitive.CollectdWin { @@ -191,6 +192,28 @@ private IList GetCommonAttributes() } AttributeValue ram = new AttributeValue(_hostName, "ram bytes", totalRAM.ToString()); attributes.Add(ram); + + try + { + var host = Dns.GetHostEntry(Dns.GetHostName()); + List addressList = new List(); + foreach (var ip in host.AddressList) + { + // Only get IP V4 addresses for now + if (ip.AddressFamily == AddressFamily.InterNetwork) + addressList.Add(ip.ToString()); + } + + string ipStr = string.Join(",", addressList.ToArray()); + + AttributeValue ipAttr = new AttributeValue(_hostName, "ip", ipStr); + attributes.Add(ipAttr); + } + catch (Exception ex) + { + Logger.Error("Failed to get IP address", ex); + } + return attributes; } From 892911e908934097f0a49f452d4ba3b3ef7f6ae8 Mon Sep 17 00:00:00 2001 From: Clift Greene Date: Thu, 10 Mar 2016 20:18:52 -0500 Subject: [PATCH 04/30] Update app.config --- src/CollectdWinService/app.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CollectdWinService/app.config b/src/CollectdWinService/app.config index 43aeed2..8f382d8 100644 --- a/src/CollectdWinService/app.config +++ b/src/CollectdWinService/app.config @@ -170,7 +170,7 @@ CollectdPluginInstance="" CollectdType="percent" CollectdTypeInstance="sqlservr_percent_processor_time" /> + CollectdPluginInstance="" CollectdType="count" CollectdTypeInstance="user_connections" /> @@ -236,4 +236,4 @@ - \ No newline at end of file + From f100a9bb3511f30aee68953637a0bd0f280c4d30 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Thu, 17 Mar 2016 17:38:27 +0000 Subject: [PATCH 05/30] PHX-3997 Corrected typo in default sql metric name --- src/CollectdWinService/config/ReadWindowsPerfCounters.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CollectdWinService/config/ReadWindowsPerfCounters.config b/src/CollectdWinService/config/ReadWindowsPerfCounters.config index 63f871e..f05fe77 100644 --- a/src/CollectdWinService/config/ReadWindowsPerfCounters.config +++ b/src/CollectdWinService/config/ReadWindowsPerfCounters.config @@ -136,7 +136,7 @@ CollectdPluginInstance="" CollectdType="percent" CollectdTypeInstance="sqlservr_percent_processor_time" /> + CollectdPluginInstance="" CollectdType="count" CollectdTypeInstance="user_connections" /> From 00a59415f97384c1b565d1685305342ed56ed427 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Thu, 31 Mar 2016 16:38:18 +0100 Subject: [PATCH 06/30] PHX-4221 - combine events of the same type into one event. PHX-4222, add event count as metric. Also simplified and enhanced filtering options to give users greater control. --- src/CollectdWinService/CollectableValue.cs | 29 +---- .../ReadWindowsEventsPlugin.cs | 104 +++++++++++------- .../WriteNetuitivePlugin.cs | 2 +- .../config/ReadWindowsEventPluginConfig.cs | 41 ++++++- .../config/ReadWindowsEvents.config | 20 +++- 5 files changed, 123 insertions(+), 73 deletions(-) diff --git a/src/CollectdWinService/CollectableValue.cs b/src/CollectdWinService/CollectableValue.cs index dc7de37..8750f00 100644 --- a/src/CollectdWinService/CollectableValue.cs +++ b/src/CollectdWinService/CollectableValue.cs @@ -137,13 +137,15 @@ internal class EventValue : CollectableValue public long Id { get; set; } public string Message { get; set; } public string Level { get; set; } + public string Title { get; set; } public long Timestamp { get; set; } - private const string JSON_FORMAT = @"{{""level"": ""{0}"", ""source"":""{1}"", ""message"":""{2}"", ""timestamp"": {3} }}"; + private const string JSON_FORMAT = @"{{""level"": ""{0}"", ""source"":""{1}"", ""title"":""{2}"", ""message"":""{3}"", ""timestamp"": {4} }}"; - public EventValue(string hostname, long timestamp, int nLevel, string message, long id) + public EventValue(string hostname, long timestamp, int nLevel, string title, string message, long id) { Level = EventValue.levelToString(nLevel); Timestamp = timestamp; + Title = title; Message = message; HostName = hostname; Id = id; @@ -154,27 +156,6 @@ public EventValue(string hostname, long timestamp, int nLevel, string message, l TypeInstanceName = ""; } - public static int levelToInt(string level) - { - switch (level) - { - case "CRITICAL": - return 1; - case "ERROR": - return 2; - case "WARN": - return 3; - case "WARNING": - return 3; - case "INFO": - return 4; - case "DEBUG": - return 5; - default: // not specified - return -1; - } - } - public static string levelToString(int level) { switch (level) @@ -210,7 +191,7 @@ public override bool Equals(object obj) public override string getJSON() { - return string.Format(JSON_FORMAT, Level, HostName, Message, Timestamp*1000); + return string.Format(JSON_FORMAT, Level, HostName, Title, Message, Timestamp*1000); } } diff --git a/src/CollectdWinService/ReadWindowsEventsPlugin.cs b/src/CollectdWinService/ReadWindowsEventsPlugin.cs index 8277cbe..7d8793b 100644 --- a/src/CollectdWinService/ReadWindowsEventsPlugin.cs +++ b/src/CollectdWinService/ReadWindowsEventsPlugin.cs @@ -13,8 +13,13 @@ internal struct EventQuery { public string log; public string source; + public int minLevel; public int maxLevel; public string filterExp; + public string title; + public int maxPerCycle; + public int minEventId; + public int maxEventId; } internal class ReadWindowsEventsPlugin : ICollectdReadPlugin @@ -45,17 +50,21 @@ public void Configure() foreach (WindowsEventConfig eventConfig in config.Events) { - int level = EventValue.levelToInt(eventConfig.MaxLevel); - EventQuery evt = new EventQuery { log = eventConfig.Log, source = eventConfig.Source, filterExp = eventConfig.FilterExp, - maxLevel = level + minLevel = eventConfig.MinLevel, + maxLevel = eventConfig.MaxLevel, + maxPerCycle = eventConfig.MaxEventsPerCycle, + title = eventConfig.Title, + minEventId = eventConfig.MinEventId, + maxEventId = eventConfig.MaxEventId + }; _events.Add(evt); - Logger.Info("Added event reader: {0}, {1}, {2}", evt.log, evt.source, evt.maxLevel); + Logger.Info("Added event reader {0}: log:{1}, source:{2}, level:{3}-{4}, ID:{5}-{6}, maxAllowed:{7}", evt.title, evt.log, evt.source, evt.minLevel, evt.maxLevel, evt.minEventId, evt.maxEventId, evt.maxPerCycle); } Logger.Info("ReadWindowsEvents plugin configured"); @@ -74,44 +83,65 @@ public void Stop() public IList Read() { IList collectableValues = new List(); + long totalEvents = 0; + long collectionTime = (long)(Util.toEpoch(DateTime.UtcNow)); + List recordIds = new List(); foreach (EventQuery eventQuery in _events) { - List records = GetEventRecords(eventQuery.maxLevel, eventQuery.log, eventQuery.source); - foreach (EventRecord record in records) - { - // Timestamp from record is machine time, not GMT - long timestamp = (long)(Util.toEpoch(record.TimeCreated.Value.ToUniversalTime())); - long id = (long)record.RecordId; - string message = record.FormatDescription(); - EventValue newevent = new EventValue(_hostName, timestamp, record.Level.Value, message, id); - - // Dedupe - bool add = true; - foreach(EventValue ev in collectableValues) { - if (ev.Equals(newevent)) - { - add = false; - break; - } - } + List records = GetEventRecords(eventQuery.minLevel, eventQuery.maxLevel, eventQuery.log, eventQuery.source); - // Filter - if (add) - { - Regex regex = new Regex(eventQuery.filterExp, RegexOptions.None); - add &= regex.IsMatch(message); - } + // Filter the events - event ID must be in target range, description must match regex and we mustn't have already read this event record ID in another query + Regex filterExp = new Regex(eventQuery.filterExp, RegexOptions.None); + List filteredRecords = records.FindAll(delegate(EventRecord record) { + return !recordIds.Contains((long)record.RecordId) && record.Id >= eventQuery.minEventId && record.Id <= eventQuery.maxEventId && filterExp.IsMatch(record.FormatDescription()); + }); + + // Add these record IDs to dedupe list so we don't capture them again in a later query + filteredRecords.ForEach(delegate(EventRecord record) { recordIds.Add((long)record.RecordId);}); - if (add) + if (filteredRecords.Count <= eventQuery.maxPerCycle) + { + foreach (EventRecord record in filteredRecords) + { + // Timestamp from record is machine time, not GMT + long timestamp = (long)(Util.toEpoch(record.TimeCreated.Value.ToUniversalTime())); + long id = (long)record.RecordId; + string message = record.FormatDescription(); + EventValue newevent = new EventValue(_hostName, timestamp, record.Level.Value, eventQuery.title, message, id); collectableValues.Add(newevent); + totalEvents++; + } + } + else + { + // Too many events - reduce to summary using attributes of the first + EventRecord record = filteredRecords[0]; + string message = string.Format("Received {0} events", filteredRecords.Count); + string title = string.Format("{0} ({1} events)", eventQuery.title, filteredRecords.Count); + EventValue newevent = new EventValue(_hostName, collectionTime, record.Level.Value, title, message, 0); + collectableValues.Add(newevent); + totalEvents += filteredRecords.Count; } } - + + // Add event count metric + MetricValue eventCountMetric = new MetricValue + { + HostName = _hostName, + PluginName = "windows_events", + PluginInstanceName = "", + TypeName = "count", + TypeInstanceName = "event_count", + Values = new double[]{totalEvents}, + FriendlyNames = new string[]{"Windows Event Count"}, + Epoch = collectionTime + }; + collectableValues.Add(eventCountMetric); return collectableValues; } // 1 = critical, 2=error, 3=warning, 4=information,5=verbose, -1=no filter - private List GetEventRecords(int level, string logName,string providerName) { + private List GetEventRecords(int minLevel, int maxLevel, string logName,string providerName) { List eventRecords = new List(); EventRecord eventRecord; string queryString; @@ -120,19 +150,11 @@ private List GetEventRecords(int level, string logName,string provi { if (providerName != null && providerName.Length > 0) { - queryString = "*"; - if (level >= 0) - queryString += String.Format("[System/Level <= {0}]", level); - - queryString += String.Format("[System/Provider/@Name = '{0}'][System/TimeCreated[timediff(@SystemTime) <= {1}]]", providerName, _interval * 1000); + queryString = String.Format("*[System[(Level >= {0}) and (Level <= {1}) and Provider/@Name = '{2}' and TimeCreated[timediff(@SystemTime) <= {3}]]]", minLevel, maxLevel, providerName, _interval * 1000); } else { - queryString = "*[System["; - if (level >= 0) - queryString += String.Format("(Level <= {0}) and ", level); - queryString += String.Format( - "TimeCreated[timediff(@SystemTime) <= {0}]]]", _interval * 1000); + queryString = String.Format("*[System[(Level >= {0}) and (Level <= {1}) and TimeCreated[timediff(@SystemTime) <= {2}]]]", minLevel, maxLevel, _interval * 1000); } EventLogQuery query = new EventLogQuery(logName, PathType.LogName, queryString); EventLogReader reader = new EventLogReader(query); diff --git a/src/CollectdWinService/WriteNetuitivePlugin.cs b/src/CollectdWinService/WriteNetuitivePlugin.cs index ce9877b..5b832a3 100644 --- a/src/CollectdWinService/WriteNetuitivePlugin.cs +++ b/src/CollectdWinService/WriteNetuitivePlugin.cs @@ -148,7 +148,7 @@ protected List ConvertEventsToIngestEvents(List events) // Format title and message string message = value.Message; - string title = value.Level + " - " + value.Message; + string title = value.Title; if (title.Length > _maxEventTitleLength) title = title.Substring(0, _maxEventTitleLength); diff --git a/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs b/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs index 869de94..6b78c3a 100644 --- a/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs +++ b/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs @@ -45,19 +45,54 @@ public String Source set { base["Source"] = value; } } - [ConfigurationProperty("MaxLevel", IsRequired = false, DefaultValue="")] - public string MaxLevel + [ConfigurationProperty("MaxLevel", IsRequired = true)] + public int MaxLevel { - get { return (string)base["MaxLevel"]; } + get { return (int)base["MaxLevel"]; } set { base["MaxLevel"] = value; } } + [ConfigurationProperty("MinLevel", IsRequired = false, DefaultValue = 1)] + public int MinLevel + { + get { return (int)base["MinLevel"]; } + set { base["MinLevel"] = value; } + } + [ConfigurationProperty("FilterExp", IsRequired = false, DefaultValue=".*")] public string FilterExp { get { return (string)base["FilterExp"]; } set { base["FilterExp"] = value; } } + + [ConfigurationProperty("Title", IsRequired = true)] + public string Title + { + get { return (string)base["Title"]; } + set { base["Title"] = value; } + } + + [ConfigurationProperty("MaxEventsPerCycle", IsRequired = false, DefaultValue=1)] + public int MaxEventsPerCycle + { + get { return (int)base["MaxEventsPerCycle"]; } + set { base["MaxEventsPerCycle"] = value; } + } + + [ConfigurationProperty("MinEventId", IsRequired = false, DefaultValue = 0)] + public int MinEventId + { + get { return (int)base["MinEventId"]; } + set { base["MinEventId"] = value; } + } + + [ConfigurationProperty("MaxEventId", IsRequired = false, DefaultValue = 65535)] + public int MaxEventId + { + get { return (int)base["MaxEventId"]; } + set { base["MaxEventId"] = value; } + } } } diff --git a/src/CollectdWinService/config/ReadWindowsEvents.config b/src/CollectdWinService/config/ReadWindowsEvents.config index 91d8d4d..8495b4d 100644 --- a/src/CollectdWinService/config/ReadWindowsEvents.config +++ b/src/CollectdWinService/config/ReadWindowsEvents.config @@ -1,11 +1,23 @@  - + + - + - - + + + + + + \ No newline at end of file From 098c4c03fc2baef83a980dec2df1ae8a38baa956 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Mon, 4 Apr 2016 11:25:06 +0100 Subject: [PATCH 07/30] PHX-4108 Added initial read of all performance counters on every refresh so that counters that need 2 reads to report give a value on the next collection cycle. Also increased service startup time to prevent warnings --- src/CollectdWinService/CollectdWinService.cs | 3 +++ src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/CollectdWinService/CollectdWinService.cs b/src/CollectdWinService/CollectdWinService.cs index 1fbd76f..51a0dd1 100644 --- a/src/CollectdWinService/CollectdWinService.cs +++ b/src/CollectdWinService/CollectdWinService.cs @@ -15,6 +15,9 @@ public CollectdWinService() protected override void OnStart(string[] args) { + // Request additional service startup time. Configuring the metrics can take a little while + RequestAdditionalTime(30000); + StartService(args); } diff --git a/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs b/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs index 43727af..b404836 100644 --- a/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs +++ b/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using NLog; using System.Text.RegularExpressions; +using System.Threading; namespace BloombergFLP.CollectdWin { @@ -137,6 +138,9 @@ public void Configure() } } } + // Wait 1 second for the two-valued counters to be ready for next incremental read - see https://msdn.microsoft.com/en-us/library/system.diagnostics.performancecounter.nextvalue(v=vs.110).aspx + Thread.Sleep(1000); + Logger.Info("ReadWindowsPerfeCounters plugin configured {0} metrics", metricCounter); } @@ -218,7 +222,10 @@ private Boolean AddPerformanceCounter(string category, string names, string inst int ix = 0; foreach (string ctr in counterList) { - metric.Counters.Add(new PerformanceCounter(category, ctr.Trim(), instance)); + PerformanceCounter perfCounter = new PerformanceCounter(category, ctr.Trim(), instance); + // Collect a value - this is needed to initialise counters that need two values + perfCounter.NextValue(); + metric.Counters.Add(perfCounter); string friendlyName = ctr.Trim(); if (instance.Length > 0) friendlyName += " (" + instance + ")"; From 7b4ba9ff7e2d59d7e9c57ac610711ac15a71d95f Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Thu, 7 Apr 2016 22:19:46 +0100 Subject: [PATCH 08/30] PHX-4283 Added IIS/.Net metrics --- .../config/ReadWindowsPerfCounters.config | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/CollectdWinService/config/ReadWindowsPerfCounters.config b/src/CollectdWinService/config/ReadWindowsPerfCounters.config index f05fe77..1cf11ed 100644 --- a/src/CollectdWinService/config/ReadWindowsPerfCounters.config +++ b/src/CollectdWinService/config/ReadWindowsPerfCounters.config @@ -130,7 +130,6 @@ - @@ -165,5 +164,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From ad61b8cb902c5ee0389caa0bfd27b1d4bae1b7e6 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Fri, 8 Apr 2016 19:51:35 +0100 Subject: [PATCH 09/30] PHX-4282 Added shutdown code support --- src/CollectdWinService/MetricsCollector.cs | 4 +- src/CollectdWinService/Util.cs | 27 ++++++++---- .../WriteNetuitivePlugin.cs | 43 +++++++++++++++---- 3 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/CollectdWinService/MetricsCollector.cs b/src/CollectdWinService/MetricsCollector.cs index 3933dad..ac486ce 100644 --- a/src/CollectdWinService/MetricsCollector.cs +++ b/src/CollectdWinService/MetricsCollector.cs @@ -216,9 +216,7 @@ private void WriteThreadProc() } } } - double writeEnd = Util.GetNow(); - Logger.Info("Written {0} values in {1:0.00}s", numValues, (writeEnd - writeStart)); - + double writeEnd = Util.GetNow(); double elapsed = writeEnd - writeStart; double revisedInterval = (_interval - elapsed) * 1000; if (revisedInterval / _interval < 0.1) diff --git a/src/CollectdWinService/Util.cs b/src/CollectdWinService/Util.cs index c726f60..f625583 100644 --- a/src/CollectdWinService/Util.cs +++ b/src/CollectdWinService/Util.cs @@ -3,6 +3,7 @@ using System.Net; using System.Threading; using NLog; +using System.Collections.Generic; namespace BloombergFLP.CollectdWin { @@ -37,18 +38,17 @@ public static string GetHostName() return (Environment.MachineName.ToLower()); } - public static string PostJson(string url, string payload) + public static KeyValuePair PostJson(string url, string payload) { -// Logger.Debug("WriteNetuitive: {0}", payload); -// Uri uri = new Uri("http://127.0.0.1:8888"); - string result = ""; + string message = ""; + int statusCode = 200; + try { - using (var client = new WebClient()) { client.Headers[HttpRequestHeader.ContentType] = "application/json"; - result = client.UploadString(url, "POST", payload); + message = client.UploadString(url, "POST", payload); } } catch (System.Net.WebException ex) @@ -61,12 +61,21 @@ public static string PostJson(string url, string payload) else { Logger.Error("Error posting payload to {0}", url, ex); - return ex.Message; + message = ex.Message; + if (ex.Response as HttpWebResponse != null) + { + // Get the actual code + statusCode = ((HttpWebResponse)ex.Response).StatusCode.GetHashCode(); + } + else + { + // use a generic client error + statusCode = 400; + } } } - return result; - + return new KeyValuePair(statusCode, message); } } } diff --git a/src/CollectdWinService/WriteNetuitivePlugin.cs b/src/CollectdWinService/WriteNetuitivePlugin.cs index 5b832a3..f2e32ec 100644 --- a/src/CollectdWinService/WriteNetuitivePlugin.cs +++ b/src/CollectdWinService/WriteNetuitivePlugin.cs @@ -23,6 +23,7 @@ internal class WriteNetuitivePlugin : ICollectdWritePlugin private string _location; private string _defaultElementType; private int _payloadSize; + private bool _enabled; public void Configure() { @@ -57,8 +58,9 @@ public void Configure() Logger.Info("Maximum payload size: {0}", _payloadSize); - _maxEventTitleLength = config.MaxEventTitleLength; + + _enabled = true; } public void Start() @@ -73,6 +75,9 @@ public void Stop() public void Write(CollectableValue value) { + if (!_enabled) + return; + Queue entry = new Queue(); entry.Enqueue(value); Write(entry); @@ -80,6 +85,10 @@ public void Write(CollectableValue value) public void Write(Queue values) { + if (!_enabled) + return; + + double writeStart = Util.GetNow(); // Split into separate lists for each ingest point List metricsAttributesAndRelations = null; @@ -102,6 +111,8 @@ public void Write(Queue values) if (eventList.Count > 0) PostEvents(eventList); + double writeEnd = Util.GetNow(); + Logger.Debug("Write took {1:0.00}s", (writeEnd - writeStart)); } protected List ConvertMetricsAttributesAndRelationsToIngestElements(List metricsAttributes) @@ -219,12 +230,14 @@ private void PostEvents(List eventList) { eventPayloads.Add(SerialiseJsonObject(ingestEvent, typeof(IngestEvent))); } - + string eventPayload = "[" + string.Join(",", eventPayloads.ToArray()) + "]"; - string res = Util.PostJson(_eventIngestUrl, eventPayload); - if (res.Length > 0) + KeyValuePair res = Util.PostJson(_eventIngestUrl, eventPayload); + + bool isOK = ProcessResponseCode(res.Key); + if (!isOK) { - Logger.Warn("Error posting events: {0}", res); + Logger.Warn("Error posting events: {0}, {1}", res.Key, res.Value); Logger.Warn("Payload: {0}", eventPayload); } } @@ -235,10 +248,11 @@ private void PostMetricsAndAttributes(List mergedIngestElementLis foreach (IngestElement ingestElement in mergedIngestElementList) { string payload = "[" + SerialiseJsonObject(ingestElement, typeof(IngestElement)) + "]"; - string res = Util.PostJson(_ingestUrl, payload); - if (res.Length > 0) + KeyValuePair res = Util.PostJson(_ingestUrl, payload); + bool isOK = ProcessResponseCode(res.Key); + if (!isOK) { - Logger.Warn("Error posting metrics/attributes: {0}", res); + Logger.Warn("Error posting metrics/attributes: {0}, {1}", res.Key, res.Value); Logger.Warn("Payload: {0}", payload); } } @@ -332,6 +346,19 @@ protected void GetIngestRelations(RelationValue value, out List relations = new List(); relations.Add(new IngestRelation(value.Fqn)); } + + protected bool ProcessResponseCode(int responseCode) + { + if (responseCode == 410) + { + // shutdown this plugin + Logger.Fatal("Received plugin shutdown code from server"); + _enabled = false; + return false; + } + else return + responseCode >=200 && responseCode < 300; + } } // ******************** DataContract objects for JSON serialisation ******************** From 244603239db68e56a91d75165b97dedd11e00a15 Mon Sep 17 00:00:00 2001 From: Jordan Kauffman Date: Tue, 12 Apr 2016 15:01:26 -0400 Subject: [PATCH 10/30] Adding CONTRIBUTING.md adding guidelines on contributing to our fork --- CONTRIBUTING.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..06ef588 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,21 @@ +Contributing to the Netuitive Windows Agent +====================================== + +### Workflow for contributing + +1. Create a branch directly in this repo or a fork (if you don't have push access). Please name branches within this repository `feature/` or `fix/description`. For example, something like `feature/upgrade_agent_0.2.3-70`. + +1. Create an issue or open a pull request (PR). If you aren't sure your PR will solve the issue or may be controversial, we're okay with you opening an issue separately and linking to it in your PR. That way, if the PR is not accepted, the issue will remain and be tracked. + +1. Close (and reference) issues by the `closes #XXX` or `fixes #XXX` notation in the commit message. Please use a descriptive, useful commit message that could be used to understand why a particular change was made. Keep pushing commits to the initial branch using `--amend`/`--rebase` as necessary, and don't mix unrelated issues in a single branch. + +1. Clean up the branch (rebase with master to synchronize, squash, edit commits, etc.) to prepare for it to be merged. + +### Merging contributions + +1. After reviewing commits for descriptive commit messages, documentation, passed continuous integration (CI) tests, version bumps, and a changelog, a project maintainer can merge. + + +### Releasing + +1. Create/update the changelog if necessary. \ No newline at end of file From 6bbbb0647a459733c82be94ec1504dd6befb97b2 Mon Sep 17 00:00:00 2001 From: Jordan Kauffman Date: Thu, 28 Apr 2016 11:38:37 -0400 Subject: [PATCH 11/30] Updating README.md committing new format to readme file --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3456096..8885166 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,12 @@ -#Netuitive-CollectdWin +Netuitive Windows Agent +======================== -CollectdWin is a MS Windows service which collects, aggregates and publishes windows performance counters and attributes. CollectdWin is similar in concept and design to Collectd (https://collectd.org). +The Netuitive Windows Agent leverages CollectdWin to collect, aggregate, and publish windows performance counters and attributes to Netuitive. It is designed to expose crucial metrics from your Windows machines and display them in a meaningful way in [Netuitive](https://http://www.netuitive.com/). The Netuitive Windows Agent is a fork of the CollectdWin project, which is similar in concept and design to [Collectd](https://collectd.org). -This version was forked to add Write Netuitive and other plugins. +See the [Netuitive Windows agent docs](https://help.netuitive.com/Content/Misc/Datasources/new_jvm_datasource.htm) or the [wiki](../../wiki) for more information, or contact Netuitive support at [support@netuitive.com](mailto:support@netuitive.com). + +Changes to CollectdWin +----------------------- + +The core functionality of CollectdWin remains the same in our fork: exposing windows performance counters and attributes for collection and monitoring. The Netuitive Windows Agent diverges from CollectdWin when considering the different plugins available. Netuitive created plugins to read Windows events and attributes as well as plugins to write to [Netuitive](https://http://www.netuitive.com/) and [StatsD](https://github.com/etsy/statsd). Netuitive also changed much of the source code to allow for various other plugins, including the Read StatsD, Write HTTP, and Write AMQP. -For more info, see the [wiki](../../wiki) From 6d3d03e2bcfcaf79b67ec2163daf7cd27935bbb7 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Mon, 2 May 2016 09:23:50 +0100 Subject: [PATCH 12/30] PHX-4221 increase detail recorded in aggregated event --- .../ReadWindowsEventsPlugin.cs | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/CollectdWinService/ReadWindowsEventsPlugin.cs b/src/CollectdWinService/ReadWindowsEventsPlugin.cs index 7d8793b..bb13a02 100644 --- a/src/CollectdWinService/ReadWindowsEventsPlugin.cs +++ b/src/CollectdWinService/ReadWindowsEventsPlugin.cs @@ -7,6 +7,7 @@ using BloombergFLP.CollectdWin; using System.Text.RegularExpressions; + namespace Netuitive.CollectdWin { internal struct EventQuery @@ -114,11 +115,39 @@ public IList Read() } else { - // Too many events - reduce to summary using attributes of the first - EventRecord record = filteredRecords[0]; - string message = string.Format("Received {0} events", filteredRecords.Count); + // Too many events - summarise by counting events by application,level and code + Dictionary detailMap = new Dictionary(); + int minLevel = 999; // used to get the most severe event in the period for the summary level + filteredRecords.ForEach(delegate(EventRecord record) { + string key = string.Format("{0} in {1} ({2})", record.LevelDisplayName, record.ProviderName, record.Id); + + if (record.Level.Value < minLevel) + minLevel = record.Level.Value; + + if (detailMap.ContainsKey(key)) + { + detailMap[key] = detailMap[key] + 1; + } + else + { + detailMap.Add(key, 1); + } + }); + + List> detailList = new List>(); + foreach (string key in detailMap.Keys) { + detailList.Add(new KeyValuePair(key, detailMap[key])); + } + detailList.Sort(delegate(KeyValuePair pair1, KeyValuePair pair2){return -pair1.Value.CompareTo(pair2.Value);}); + + string[] messageLines = new string[detailList.Count]; + + int ix = 0; + foreach(KeyValuePair pair in detailList) { + messageLines[ix++] = pair.Value + " x " + pair.Key; + } string title = string.Format("{0} ({1} events)", eventQuery.title, filteredRecords.Count); - EventValue newevent = new EventValue(_hostName, collectionTime, record.Level.Value, title, message, 0); + EventValue newevent = new EventValue(_hostName, collectionTime, minLevel, title, String.Join(", ", messageLines), 0); collectableValues.Add(newevent); totalEvents += filteredRecords.Count; } From d3ee9c7e070715ccf15af95149c9e2a228342869 Mon Sep 17 00:00:00 2001 From: Jordan Kauffman Date: Tue, 3 May 2016 14:43:47 -0400 Subject: [PATCH 13/30] Readme update, contributing.md format update updating links in the readme, minor changes; applying new format to contributing --- CONTRIBUTING.md | 73 +++++++++++++++++++++++++++++++++++++++++-------- README.md | 6 ++-- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 06ef588..ac75e5c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,21 +1,70 @@ Contributing to the Netuitive Windows Agent -====================================== +========================================== +Contributions are welcome and can be represented in many different ways as noted below. Help is +greatly appreciated and credit will always be given. -### Workflow for contributing + +Types of Contributions +------------------------------ + +### Reporting Bugs +Report bugs on [the issues page](https://github.com/Netuitive/collectdwin/issues). +With your bug report, please include: +- Your operating system name and version. +- Any details about your local setup that might be helpful in troubleshooting the issue. +- Detailed steps to reproduce the bug. + +### Fixing bugs +Find bugs at [the issues page](https://github.com/Netuitive/collectdwin/issues). Anything tagged with +"bug" is open to be fixed. +With your fix, please include: +- The issue number +- A detailed commit message + +### Implementing Features +Find features at [the issues page](https://github.com/Netuitive/collectdwin/issues). Anything tagged +with "feature" is open to be implemented. +With your feature, please include: +- The issue number +- A detailed commit message + +### Writing Documentation +The Netuitive Windows Agent can always use documentation (more documentation is always better!). +Please document your features or usage as part of the official docs/wiki, in docstrings, +in blog posts, articles, or wherever you see fit. + +### Submitting Feedback +File an issue at [the issues page](https://github.com/Netuitive/collectdwin/issues). +If you are proposing a feature: +- Explain how it would work in detail +- Keep the scope as narrow as possible to make it easier to implement + +Workflow +------------------------------ + +1. Create a branch directly in this repo or a fork (if you don't have push access). Please name +branches within this repository `feature/` or `fix/description`. For example, +something like `feature/upgrade_agent_0.2.3-70`. -1. Create a branch directly in this repo or a fork (if you don't have push access). Please name branches within this repository `feature/` or `fix/description`. For example, something like `feature/upgrade_agent_0.2.3-70`. +1. Create an issue or open a pull request (PR). If you aren't sure your PR will solve the issue +or may be controversial, we're okay with you opening an issue separately and linking to it in +your PR. That way, if the PR is not accepted, the issue will remain and be tracked. -1. Create an issue or open a pull request (PR). If you aren't sure your PR will solve the issue or may be controversial, we're okay with you opening an issue separately and linking to it in your PR. That way, if the PR is not accepted, the issue will remain and be tracked. +1. Clone the fork/branch locally. + +1. Close (and reference) issues by the `closes #XXX` or `fixes #XXX` notation in the commit +message. Please use a descriptive, useful commit message that could be used to understand why a +particular change was made. + +1. Keep pushing commits to the initial branch using `--amend`/`--rebase` as necessary. Don't mix +unrelated issues in a single branch. -1. Close (and reference) issues by the `closes #XXX` or `fixes #XXX` notation in the commit message. Please use a descriptive, useful commit message that could be used to understand why a particular change was made. Keep pushing commits to the initial branch using `--amend`/`--rebase` as necessary, and don't mix unrelated issues in a single branch. +1. Clean up the branch (rebase with master to synchronize, squash, edit commits, test, etc.) to +prepare for it to be merged. -1. Clean up the branch (rebase with master to synchronize, squash, edit commits, etc.) to prepare for it to be merged. +1. If you didn't open a pull request already, do so now. -### Merging contributions - -1. After reviewing commits for descriptive commit messages, documentation, passed continuous integration (CI) tests, version bumps, and a changelog, a project maintainer can merge. - - -### Releasing +1. After reviewing your commits for documentation, passed continuous integration (CI) tests, +version bumps, changelogs, and good, descriptive commit messages, a project maintainer can merge your request. 1. Create/update the changelog if necessary. \ No newline at end of file diff --git a/README.md b/README.md index 8885166..aaae300 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ Netuitive Windows Agent ======================== -The Netuitive Windows Agent leverages CollectdWin to collect, aggregate, and publish windows performance counters and attributes to Netuitive. It is designed to expose crucial metrics from your Windows machines and display them in a meaningful way in [Netuitive](https://http://www.netuitive.com/). The Netuitive Windows Agent is a fork of the CollectdWin project, which is similar in concept and design to [Collectd](https://collectd.org). +The Netuitive Windows Agent leverages CollectdWin to collect, aggregate, and publish windows performance counters and attributes to Netuitive. It is designed to expose crucial metrics from your Windows machines and display them in a meaningful way in [Netuitive](https://http://www.netuitive.com/). -See the [Netuitive Windows agent docs](https://help.netuitive.com/Content/Misc/Datasources/new_jvm_datasource.htm) or the [wiki](../../wiki) for more information, or contact Netuitive support at [support@netuitive.com](mailto:support@netuitive.com). +See the [Netuitive Windows agent docs](https://help.netuitive.com/Content/Misc/Datasources/Windows/new_windows_datasource.htm) or the [wiki](../../wiki) for more information, or contact Netuitive support at [support@netuitive.com](mailto:support@netuitive.com). Changes to CollectdWin ----------------------- -The core functionality of CollectdWin remains the same in our fork: exposing windows performance counters and attributes for collection and monitoring. The Netuitive Windows Agent diverges from CollectdWin when considering the different plugins available. Netuitive created plugins to read Windows events and attributes as well as plugins to write to [Netuitive](https://http://www.netuitive.com/) and [StatsD](https://github.com/etsy/statsd). Netuitive also changed much of the source code to allow for various other plugins, including the Read StatsD, Write HTTP, and Write AMQP. +The base functionality of CollectdWin remains in our fork: exposing windows performance counters for collection and monitoring. The Netuitive Windows Agent diverges from CollectdWin by extending the collection to non-numeric values such as attributes, events, and relationships. Netuitive created plugins to read Windows events and attributes as well as plugins to write to [Netuitive](https://www.netuitive.com/) and [StatsD](https://github.com/etsy/statsd). Netuitive also changed the underlying framework to support collection and representation of elements of different types and metrics from remote sources. From 41d157a3d366346e0c71b98c86fc0229461b5021 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Thu, 12 May 2016 09:52:19 +0100 Subject: [PATCH 14/30] Reduced log level for missing metrics and attributes from error to warning. The metrics may be missing if a transient process stops and we don't want events for this --- src/CollectdWinService/ReadWindowsAttributesPlugin.cs | 6 +++--- src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs index 0bd04c7..db3fd96 100644 --- a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs +++ b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs @@ -93,7 +93,7 @@ public IList Read() } catch (Exception ex) { - Logger.Error(string.Format("Failed to collect attribute: {0}", attribute.variableName), ex); + Logger.Warn(string.Format("Failed to collect attribute: {0}", attribute.variableName), ex); } } return collectedValueList; @@ -188,7 +188,7 @@ private IList GetCommonAttributes() } catch (Exception ex) { - Logger.Error("Failed to get system memory", ex); + Logger.Warn("Failed to get system memory", ex); } AttributeValue ram = new AttributeValue(_hostName, "ram bytes", totalRAM.ToString()); attributes.Add(ram); @@ -211,7 +211,7 @@ private IList GetCommonAttributes() } catch (Exception ex) { - Logger.Error("Failed to get IP address", ex); + Logger.Warn("Failed to get IP address", ex); } return attributes; diff --git a/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs b/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs index b404836..f1bb230 100644 --- a/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs +++ b/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs @@ -197,7 +197,7 @@ public IList Read() } catch (Exception ex) { - Logger.Error(string.Format("Failed to collect metric: {0}, {1}, {2}", metric.Category, metric.Instance, metric.CounterName), ex); + Logger.Warn(string.Format("Failed to collect metric: {0}, {1}, {2}", metric.Category, metric.Instance, metric.CounterName), ex); } } return (metricValueList); From f03c1de045e4168f3547332d6d3e8e0c9e79a22d Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Thu, 12 May 2016 09:54:40 +0100 Subject: [PATCH 15/30] Added interval multiplier to ReadWindowsEventsPlugin to allow longer collection cycle to reduce overall event count. Defaults to 5 minutes --- .../ReadWindowsEventsPlugin.cs | 48 ++++++++++++------- .../config/ReadWindowsEventPluginConfig.cs | 7 +++ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/src/CollectdWinService/ReadWindowsEventsPlugin.cs b/src/CollectdWinService/ReadWindowsEventsPlugin.cs index bb13a02..996d9c1 100644 --- a/src/CollectdWinService/ReadWindowsEventsPlugin.cs +++ b/src/CollectdWinService/ReadWindowsEventsPlugin.cs @@ -29,6 +29,8 @@ internal class ReadWindowsEventsPlugin : ICollectdReadPlugin private readonly IList _events; private string _hostName; private int _interval; + private int _intervalMultiplier; + private long _intervalCounter; public ReadWindowsEventsPlugin() { @@ -46,9 +48,10 @@ public void Configure() _interval = baseConfig.GeneralSettings.Interval; _hostName = Util.GetHostName(); + _intervalMultiplier = config.IntervalMultiplier; + _intervalCounter = 0; _events.Clear(); - foreach (WindowsEventConfig eventConfig in config.Events) { EventQuery evt = new EventQuery @@ -83,9 +86,14 @@ public void Stop() public IList Read() { + // Only collect events and event metric every nth interval + if (_intervalCounter++ % _intervalMultiplier != 0) + return new List(); + IList collectableValues = new List(); long totalEvents = 0; long collectionTime = (long)(Util.toEpoch(DateTime.UtcNow)); + List recordIds = new List(); foreach (EventQuery eventQuery in _events) { @@ -93,12 +101,13 @@ public IList Read() // Filter the events - event ID must be in target range, description must match regex and we mustn't have already read this event record ID in another query Regex filterExp = new Regex(eventQuery.filterExp, RegexOptions.None); - List filteredRecords = records.FindAll(delegate(EventRecord record) { + List filteredRecords = records.FindAll(delegate(EventRecord record) + { return !recordIds.Contains((long)record.RecordId) && record.Id >= eventQuery.minEventId && record.Id <= eventQuery.maxEventId && filterExp.IsMatch(record.FormatDescription()); }); // Add these record IDs to dedupe list so we don't capture them again in a later query - filteredRecords.ForEach(delegate(EventRecord record) { recordIds.Add((long)record.RecordId);}); + filteredRecords.ForEach(delegate(EventRecord record) { recordIds.Add((long)record.RecordId); }); if (filteredRecords.Count <= eventQuery.maxPerCycle) { @@ -113,12 +122,13 @@ public IList Read() totalEvents++; } } - else + else { // Too many events - summarise by counting events by application,level and code Dictionary detailMap = new Dictionary(); int minLevel = 999; // used to get the most severe event in the period for the summary level - filteredRecords.ForEach(delegate(EventRecord record) { + filteredRecords.ForEach(delegate(EventRecord record) + { string key = string.Format("{0} in {1} ({2})", record.LevelDisplayName, record.ProviderName, record.Id); if (record.Level.Value < minLevel) @@ -126,7 +136,7 @@ public IList Read() if (detailMap.ContainsKey(key)) { - detailMap[key] = detailMap[key] + 1; + detailMap[key] = detailMap[key] + 1; } else { @@ -134,16 +144,18 @@ public IList Read() } }); - List> detailList = new List>(); - foreach (string key in detailMap.Keys) { - detailList.Add(new KeyValuePair(key, detailMap[key])); + List> detailList = new List>(); + foreach (string key in detailMap.Keys) + { + detailList.Add(new KeyValuePair(key, detailMap[key])); } - detailList.Sort(delegate(KeyValuePair pair1, KeyValuePair pair2){return -pair1.Value.CompareTo(pair2.Value);}); + detailList.Sort(delegate(KeyValuePair pair1, KeyValuePair pair2) { return -pair1.Value.CompareTo(pair2.Value); }); string[] messageLines = new string[detailList.Count]; int ix = 0; - foreach(KeyValuePair pair in detailList) { + foreach (KeyValuePair pair in detailList) + { messageLines[ix++] = pair.Value + " x " + pair.Key; } string title = string.Format("{0} ({1} events)", eventQuery.title, filteredRecords.Count); @@ -151,8 +163,8 @@ public IList Read() collectableValues.Add(newevent); totalEvents += filteredRecords.Count; } - } - + } + // Add event count metric MetricValue eventCountMetric = new MetricValue { @@ -161,17 +173,19 @@ public IList Read() PluginInstanceName = "", TypeName = "count", TypeInstanceName = "event_count", - Values = new double[]{totalEvents}, - FriendlyNames = new string[]{"Windows Event Count"}, + Values = new double[] { totalEvents }, + FriendlyNames = new string[] { "Windows Event Count" }, Epoch = collectionTime }; collectableValues.Add(eventCountMetric); + return collectableValues; } // 1 = critical, 2=error, 3=warning, 4=information,5=verbose, -1=no filter private List GetEventRecords(int minLevel, int maxLevel, string logName,string providerName) { List eventRecords = new List(); + long eventInterval = _intervalMultiplier * _interval * 1000; EventRecord eventRecord; string queryString; @@ -179,11 +193,11 @@ private List GetEventRecords(int minLevel, int maxLevel, string log { if (providerName != null && providerName.Length > 0) { - queryString = String.Format("*[System[(Level >= {0}) and (Level <= {1}) and Provider/@Name = '{2}' and TimeCreated[timediff(@SystemTime) <= {3}]]]", minLevel, maxLevel, providerName, _interval * 1000); + queryString = String.Format("*[System[(Level >= {0}) and (Level <= {1}) and Provider/@Name = '{2}' and TimeCreated[timediff(@SystemTime) <= {3}]]]", minLevel, maxLevel, providerName, eventInterval); } else { - queryString = String.Format("*[System[(Level >= {0}) and (Level <= {1}) and TimeCreated[timediff(@SystemTime) <= {2}]]]", minLevel, maxLevel, _interval * 1000); + queryString = String.Format("*[System[(Level >= {0}) and (Level <= {1}) and TimeCreated[timediff(@SystemTime) <= {2}]]]", minLevel, maxLevel, eventInterval); } EventLogQuery query = new EventLogQuery(logName, PathType.LogName, queryString); EventLogReader reader = new EventLogReader(query); diff --git a/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs b/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs index 6b78c3a..ebb1d8b 100644 --- a/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs +++ b/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs @@ -12,6 +12,13 @@ public WindowsEventCollection Events get { return (WindowsEventCollection)base["Events"]; } set { base["Events"] = value; } } + + [ConfigurationProperty("IntervalMultiplier", IsRequired = false, DefaultValue = 5)] + public int IntervalMultiplier + { + get { return (int)base["IntervalMultiplier"]; } + set { base["IntervalMultiplier"] = value; } + } } public sealed class WindowsEventCollection : ConfigurationElementCollection From abbc5e69ef43b428c8c8fe1bf4302641176ff7a8 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Thu, 12 May 2016 09:56:58 +0100 Subject: [PATCH 16/30] Made regex Instance expressions require an exact match where given --- .../config/ReadWindowsPerfCounters.config | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/CollectdWinService/config/ReadWindowsPerfCounters.config b/src/CollectdWinService/config/ReadWindowsPerfCounters.config index 1cf11ed..7897748 100644 --- a/src/CollectdWinService/config/ReadWindowsPerfCounters.config +++ b/src/CollectdWinService/config/ReadWindowsPerfCounters.config @@ -55,7 +55,7 @@ - - - - - - From 4a93d9f21e297a489a5e553a86045042c90ddd54 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Tue, 19 Jul 2016 12:19:37 +0100 Subject: [PATCH 17/30] PHX-5183 catch parse exception when getting EC2 metadata in case behind a proxy --- src/CollectdWinService/ReadWindowsAttributesPlugin.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs index db3fd96..0d2c7ed 100644 --- a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs +++ b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs @@ -151,6 +151,10 @@ private IList GetEC2Metadata() { Logger.Warn("Failed to get EC2 instance metadata. If this server is not an EC2 update the ReadWindowsAttributes.config file to disable collection.", ex); } + catch (Exception ex) + { + Logger.Warn("Failed to process EC2 instance metadata. If this server is not an EC2 update the ReadWindowsAttributes.config file to disable collection.", ex); + } return values; } From eb426760f180761d863506a79be92a12b656f785 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Tue, 19 Jul 2016 12:47:57 +0100 Subject: [PATCH 18/30] PHX-5202 corrected logging of write time --- src/CollectdWinService/WriteNetuitivePlugin.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CollectdWinService/WriteNetuitivePlugin.cs b/src/CollectdWinService/WriteNetuitivePlugin.cs index f2e32ec..aafa113 100644 --- a/src/CollectdWinService/WriteNetuitivePlugin.cs +++ b/src/CollectdWinService/WriteNetuitivePlugin.cs @@ -112,7 +112,7 @@ public void Write(Queue values) PostEvents(eventList); double writeEnd = Util.GetNow(); - Logger.Debug("Write took {1:0.00}s", (writeEnd - writeStart)); + Logger.Info("Write took {0:0.00}s", (writeEnd - writeStart)); } protected List ConvertMetricsAttributesAndRelationsToIngestElements(List metricsAttributes) From a8783518f60179af75fd78dc59c3f264b6a5a94b Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Tue, 19 Jul 2016 12:49:04 +0100 Subject: [PATCH 19/30] Typo --- src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs b/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs index f1bb230..41ee07d 100644 --- a/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs +++ b/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs @@ -146,7 +146,7 @@ public void Configure() public void Start() { - Logger.Info("ReadWindowsPerfCountesr plugin started"); + Logger.Info("ReadWindowsPerfCounters plugin started"); } public void Stop() From 133bc59a4e8e25facfbc020a070b571bda7e801d Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Tue, 19 Jul 2016 13:05:23 +0100 Subject: [PATCH 20/30] PHX-3009 added human readably ram attribute --- .../ReadWindowsAttributesPlugin.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs index 0d2c7ed..7ab3814 100644 --- a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs +++ b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs @@ -159,6 +159,17 @@ private IList GetEC2Metadata() return values; } + static String BytesToString(long byteCount) + { + string[] suf = { "B", "KB", "MB", "GB", "TB", "PB", "EB" }; //Longs run out around EB + if (byteCount == 0) + return "0" + suf[0]; + long bytes = Math.Abs(byteCount); + int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024))); + double num = Math.Round(bytes / Math.Pow(1024, place), 1); + return (Math.Sign(byteCount) * num).ToString() + suf[place]; + } + private IList GetCommonAttributes() { // Return standard attributes @@ -194,7 +205,9 @@ private IList GetCommonAttributes() { Logger.Warn("Failed to get system memory", ex); } - AttributeValue ram = new AttributeValue(_hostName, "ram bytes", totalRAM.ToString()); + AttributeValue ramBytes = new AttributeValue(_hostName, "ram bytes", totalRAM.ToString()); + attributes.Add(ramBytes); + AttributeValue ram = new AttributeValue(_hostName, "ram", BytesToString(totalRAM)); attributes.Add(ram); try From d918f69543d04c1dc6312e5466d1c36facca5e9c Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Tue, 19 Jul 2016 13:13:15 +0100 Subject: [PATCH 21/30] Made ram suffix consistent with server element --- .../ReadWindowsAttributesPlugin.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs index 7ab3814..5839142 100644 --- a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs +++ b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs @@ -159,15 +159,15 @@ private IList GetEC2Metadata() return values; } - static String BytesToString(long byteCount) + private String BytesToString(long numBytes) { - string[] suf = { "B", "KB", "MB", "GB", "TB", "PB", "EB" }; //Longs run out around EB - if (byteCount == 0) - return "0" + suf[0]; - long bytes = Math.Abs(byteCount); + string[] suffix = { " B", " KB", " MB", " GB", " TB", " PB"}; + if (numBytes == 0) + return "0" + suffix[0]; + long bytes = Math.Abs(numBytes); int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024))); double num = Math.Round(bytes / Math.Pow(1024, place), 1); - return (Math.Sign(byteCount) * num).ToString() + suf[place]; + return (Math.Sign(numBytes) * num).ToString() + suffix[place]; } private IList GetCommonAttributes() From b16cf46a97a4fd2e8f94878fe1305bf937fabb24 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Tue, 19 Jul 2016 14:02:46 +0100 Subject: [PATCH 22/30] PHX-5224 Added configuration option to disable IP address attribute --- .../ReadWindowsAttributesPlugin.cs | 38 ++++++++++--------- .../ReadWindowsAttributesPluginConfig.cs | 8 ++++ 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs index 5839142..eb8aada 100644 --- a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs +++ b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs @@ -27,6 +27,7 @@ internal class ReadWindowsAttributesPlugin : ICollectdReadPlugin private readonly IList _attributes; private string _hostName; private bool _readEC2InstanceMetadata; + private bool _readIPAddress; public ReadWindowsAttributesPlugin() { @@ -43,6 +44,7 @@ public void Configure() _hostName = Util.GetHostName(); _readEC2InstanceMetadata = config.ReadEC2InstanceMetadata; + _readIPAddress = config.ReadIPAddress; _attributes.Clear(); foreach (EnvironmentVariableConfig attr in config.EnvironmentVariables) @@ -210,27 +212,29 @@ private IList GetCommonAttributes() AttributeValue ram = new AttributeValue(_hostName, "ram", BytesToString(totalRAM)); attributes.Add(ram); - try + if (_readIPAddress) { - var host = Dns.GetHostEntry(Dns.GetHostName()); - List addressList = new List(); - foreach (var ip in host.AddressList) + try { - // Only get IP V4 addresses for now - if (ip.AddressFamily == AddressFamily.InterNetwork) - addressList.Add(ip.ToString()); + var host = Dns.GetHostEntry(Dns.GetHostName()); + List addressList = new List(); + foreach (var ip in host.AddressList) + { + // Only get IP V4 addresses for now + if (ip.AddressFamily == AddressFamily.InterNetwork) + addressList.Add(ip.ToString()); + } + + string ipStr = string.Join(",", addressList.ToArray()); + + AttributeValue ipAttr = new AttributeValue(_hostName, "ip", ipStr); + attributes.Add(ipAttr); + } + catch (Exception ex) + { + Logger.Warn("Failed to get IP address", ex); } - - string ipStr = string.Join(",", addressList.ToArray()); - - AttributeValue ipAttr = new AttributeValue(_hostName, "ip", ipStr); - attributes.Add(ipAttr); - } - catch (Exception ex) - { - Logger.Warn("Failed to get IP address", ex); } - return attributes; } diff --git a/src/CollectdWinService/config/ReadWindowsAttributesPluginConfig.cs b/src/CollectdWinService/config/ReadWindowsAttributesPluginConfig.cs index 5b5a323..c21a2f6 100644 --- a/src/CollectdWinService/config/ReadWindowsAttributesPluginConfig.cs +++ b/src/CollectdWinService/config/ReadWindowsAttributesPluginConfig.cs @@ -19,6 +19,14 @@ public Boolean ReadEC2InstanceMetadata get { return (Boolean)base["ReadEC2InstanceMetadata"]; } set { base["ReadEC2InstanceMetadata"] = value; } } + + [ConfigurationProperty("ReadIPAddress", IsRequired = false, DefaultValue = true)] + public Boolean ReadIPAddress + { + get { return (Boolean)base["ReadIPAddress"]; } + set { base["ReadIPAddress"] = value; } + } + } public class WindowsEnvironmentVariableCollection : ConfigurationElementCollection From 84b6edcfba4a32644069e7ef16bb6459f97d7ae3 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Tue, 19 Jul 2016 14:25:50 +0100 Subject: [PATCH 23/30] PHX-5226 initialise event tags with empty array --- src/CollectdWinService/WriteNetuitivePlugin.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CollectdWinService/WriteNetuitivePlugin.cs b/src/CollectdWinService/WriteNetuitivePlugin.cs index aafa113..fb784a2 100644 --- a/src/CollectdWinService/WriteNetuitivePlugin.cs +++ b/src/CollectdWinService/WriteNetuitivePlugin.cs @@ -522,6 +522,7 @@ public IngestEvent(string type, string source, string title, long timestamp) this.source = source; this.title = title; this.timestamp = timestamp; + this.tags = new List(); } public void setData(IngestEventData data) From 6a01a7f00ad45a981ce9a7334f30ca2085efe020 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Wed, 20 Jul 2016 15:01:34 +0100 Subject: [PATCH 24/30] Upped build version --- src/CollectdWinService/Properties/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CollectdWinService/Properties/AssemblyInfo.cs b/src/CollectdWinService/Properties/AssemblyInfo.cs index 48abb6c..5549b38 100644 --- a/src/CollectdWinService/Properties/AssemblyInfo.cs +++ b/src/CollectdWinService/Properties/AssemblyInfo.cs @@ -35,4 +35,4 @@ // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.6.*")] +[assembly: AssemblyVersion("0.7.*")] From 970f60218655998043919781ce5a2559ea53fdd8 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Wed, 20 Jul 2016 15:10:29 +0100 Subject: [PATCH 25/30] Another version numbering test --- src/CollectdWinService/Properties/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CollectdWinService/Properties/AssemblyInfo.cs b/src/CollectdWinService/Properties/AssemblyInfo.cs index 5549b38..c1a23a2 100644 --- a/src/CollectdWinService/Properties/AssemblyInfo.cs +++ b/src/CollectdWinService/Properties/AssemblyInfo.cs @@ -35,4 +35,4 @@ // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.7.*")] +[assembly: AssemblyVersion("0.7.1.*")] From c9944ef8f64eeebe33791afdff2485601019d6af Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Wed, 20 Jul 2016 15:52:47 +0100 Subject: [PATCH 26/30] Added Event IDs to error logs for visibility in EventViewer --- .../CollectdWinService.csproj | 1 + src/CollectdWinService/ErrorCodes.cs | 16 +++++++ src/CollectdWinService/MetricsCollector.cs | 44 +++++++++++++++---- src/CollectdWinService/PluginRegistry.cs | 8 +++- .../ReadWindowsEventsPlugin.cs | 5 ++- .../ReadWindowsPerfCountersPlugin.cs | 36 ++++++++++++--- src/CollectdWinService/StatsdListener.cs | 10 ++++- src/CollectdWinService/TypesDB.cs | 20 ++++++--- src/CollectdWinService/Util.cs | 7 ++- src/CollectdWinService/WriteAmqpPlugin.cs | 10 ++++- src/CollectdWinService/WriteHTTPPlugin.cs | 5 ++- src/CollectdWinService/WriteStatsdPlugin.cs | 5 ++- src/CollectdWinService/app.config | 3 +- 13 files changed, 139 insertions(+), 31 deletions(-) create mode 100644 src/CollectdWinService/ErrorCodes.cs diff --git a/src/CollectdWinService/CollectdWinService.csproj b/src/CollectdWinService/CollectdWinService.csproj index 0d5c21a..066a5be 100644 --- a/src/CollectdWinService/CollectdWinService.csproj +++ b/src/CollectdWinService/CollectdWinService.csproj @@ -122,6 +122,7 @@ + diff --git a/src/CollectdWinService/ErrorCodes.cs b/src/CollectdWinService/ErrorCodes.cs new file mode 100644 index 0000000..5574cbc --- /dev/null +++ b/src/CollectdWinService/ErrorCodes.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BloombergFLP.CollectdWin +{ + public static class ErrorCodes + { + public static readonly int ERROR_READ_EXCEEDED_CYCLE_TIME = 1; + public static readonly int ERROR_WRITE_EXCEEDED_CYCLE_TIME = 2; + public static readonly int ERROR_EXCEEDED_MAX_QUEUE_LENGTH = 3; + public static readonly int ERROR_UNHANDLED_EXCEPTION = 4; + public static readonly int ERROR_CONFIGURATION_EXCEPTION = 5; + } +} diff --git a/src/CollectdWinService/MetricsCollector.cs b/src/CollectdWinService/MetricsCollector.cs index ac486ce..2246078 100644 --- a/src/CollectdWinService/MetricsCollector.cs +++ b/src/CollectdWinService/MetricsCollector.cs @@ -28,7 +28,9 @@ public MetricsCollector() var config = ConfigurationManager.GetSection("CollectdWinConfig") as CollectdWinConfig; if (config == null) { - Logger.Error("Cannot get configuration section"); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Cannot get configuration section"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); return; } @@ -140,7 +142,10 @@ private void ReadThreadProc() _collectedValueQueue.Dequeue(); if ((++numMetricsDropped%1000) == 0) { - Logger.Error("Number of metrics dropped : {0}", numMetricsDropped); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exceeded max queue length"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_EXCEEDED_MAX_QUEUE_LENGTH); + Logger.Log(logEvent); + Logger.Warn("Number of metrics dropped : {0}", numMetricsDropped); } } } @@ -151,10 +156,17 @@ private void ReadThreadProc() double revisedInterval = (_interval - elapsed) * 1000; if (revisedInterval / _interval < 0.1) { - Logger.Error("Read thread took {0} seconds out of {1} second cycle", elapsed, _interval); + Logger.Warn("Read thread took {0} seconds out of {1} second cycle", elapsed, _interval); } if (revisedInterval > 0) Thread.Sleep((int)revisedInterval); + else + { + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Read thread exceeded cycle time"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_READ_EXCEEDED_CYCLE_TIME); + Logger.Log(logEvent); + } + } catch (ThreadInterruptedException) { @@ -162,7 +174,10 @@ private void ReadThreadProc() } catch (Exception exp) { - Logger.Error("ReadThreadProc() got exception : ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception in ReadThreadProc()"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); Thread.Sleep(_interval * 1000); } } @@ -219,12 +234,19 @@ private void WriteThreadProc() double writeEnd = Util.GetNow(); double elapsed = writeEnd - writeStart; double revisedInterval = (_interval - elapsed) * 1000; + if (revisedInterval / _interval < 0.1) { - Logger.Error("Write thread took {0} seconds out of {1} second cycle", elapsed, _interval); + Logger.Warn("Write thread took {0} seconds out of {1} second cycle", elapsed, _interval); } - if (revisedInterval > 0) + if (revisedInterval >= 0) Thread.Sleep((int)revisedInterval); + else + { + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Write thread exceeded cycle time"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_WRITE_EXCEEDED_CYCLE_TIME); + Logger.Log(logEvent); + } } catch (ThreadInterruptedException) @@ -233,7 +255,10 @@ private void WriteThreadProc() } catch (Exception exp) { - Logger.Error("WriteThreadProc() got exception : ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception in WriteThreadProc()"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); Thread.Sleep(_interval * 1000); } } @@ -256,7 +281,10 @@ private void AggregatorThreadProc() } catch (Exception exp) { - Logger.Error("AggregatorThreadProc() got exception : ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception in AggregatorThreadProc()"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } } Logger.Trace("AggregatorThreadProc() return"); diff --git a/src/CollectdWinService/PluginRegistry.cs b/src/CollectdWinService/PluginRegistry.cs index 73ecc05..818c865 100644 --- a/src/CollectdWinService/PluginRegistry.cs +++ b/src/CollectdWinService/PluginRegistry.cs @@ -18,7 +18,9 @@ public PluginRegistry() var config = ConfigurationManager.GetSection("CollectdWinConfig") as CollectdWinConfig; if (config == null) { - Logger.Error("Cannot get configuration section"); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Cannot get configuration section"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); return; } foreach (PluginConfig pluginConfig in @@ -37,7 +39,9 @@ public IList CreatePlugins() Type classType = Type.GetType(entry.Value); if (classType == null) { - Logger.Error("Cannot create plugin:{0}, class:{1}", entry.Key, entry.Value); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("Cannot create plugin:{0}, class:{1}", entry.Key, entry.Value)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); continue; } var plugin = (ICollectdPlugin) Activator.CreateInstance(classType); diff --git a/src/CollectdWinService/ReadWindowsEventsPlugin.cs b/src/CollectdWinService/ReadWindowsEventsPlugin.cs index 996d9c1..26695ea 100644 --- a/src/CollectdWinService/ReadWindowsEventsPlugin.cs +++ b/src/CollectdWinService/ReadWindowsEventsPlugin.cs @@ -216,7 +216,10 @@ private List GetEventRecords(int minLevel, int maxLevel, string log } catch (Exception ex) { - Logger.Error(ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Unhandled Exception in ReadWindowsEventsPlugin"); + logEvent.Exception = ex; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } return eventRecords; diff --git a/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs b/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs index 41ee07d..245fb2e 100644 --- a/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs +++ b/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs @@ -92,18 +92,30 @@ public void Configure() } catch (ArgumentException ex) { - Logger.Error(string.Format("Failed to parse instance regular expression: category={0}, instance={1}, counter={2}", counter.Category, counter.Instance, counter.Name), ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); + Logger.Warn(string.Format("Failed to parse instance regular expression: category={0}, instance={1}, counter={2}", counter.Category, counter.Instance, counter.Name), ex); } catch (InvalidOperationException ex) { if (ex.Message.ToLower().Contains("category does not exist")) { Logger.Warn(string.Format("Performance Counter not added: Category does not exist: {0}", counter.Category)); - } else - Logger.Error(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", counter.Category, counter.Instance, counter.Name), ex); + } + else + { + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); + Logger.Warn(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", counter.Category, counter.Instance, counter.Name), ex); + } } catch (Exception ex) { - Logger.Error(string.Format("Could not initialise performance counter category: {0}", counter.Category), ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); + Logger.Warn(string.Format("Could not initialise performance counter category: {0}", counter.Category), ex); } if (instances.Length == 0) { @@ -249,13 +261,23 @@ private Boolean AddPerformanceCounter(string category, string names, string inst { if (ex.Message.ToLower().Contains("category does not exist")) { Logger.Warn(string.Format("Performance Counter not added: Category does not exist: {0}", category)); - } else - Logger.Error(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", category, instance, names), ex); + } + else + { + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Could not initialise performance counter"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); + Logger.Warn(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", category, instance, names), ex); + return false; + } return false; } catch (Exception ex) { - Logger.Error(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", category, instance, names), ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Could not initialise performance counter"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); + Logger.Warn(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", category, instance, names), ex); return false; } } diff --git a/src/CollectdWinService/StatsdListener.cs b/src/CollectdWinService/StatsdListener.cs index 9057d90..11baadb 100644 --- a/src/CollectdWinService/StatsdListener.cs +++ b/src/CollectdWinService/StatsdListener.cs @@ -36,7 +36,10 @@ private void BindSocket() } catch (Exception exp) { - Logger.Error("BindSocket failed: ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "BindSocket failed"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } if (_socket.IsBound) break; @@ -53,7 +56,10 @@ private void CloseSocket() } catch (Exception exp) { - Logger.Error("CloseSocket failed: ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "CloseSocket failed"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } } diff --git a/src/CollectdWinService/TypesDB.cs b/src/CollectdWinService/TypesDB.cs index e550013..77e058b 100644 --- a/src/CollectdWinService/TypesDB.cs +++ b/src/CollectdWinService/TypesDB.cs @@ -123,14 +123,18 @@ public void Load() Match match = dataSetRegex.Match(line); if (match.Groups.Count < 3) { - Logger.Error("types.db: invalid data set [{0}]", line); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("types.db: invalid data set [{0}]", line)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); continue; } string dataSetName = match.Groups[1].Value; MatchCollection matches = dataSourceRegex.Matches(line); if (matches.Count < 1) { - Logger.Error("types.db: invalid data source [{0}]", line); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("types.db: invalid data source [{0}]", line)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); continue; } var dataSourceList = new List(); @@ -138,7 +142,9 @@ public void Load() { if (m.Groups.Count != 5) { - Logger.Error("types.db: cannot parse data source [{0}]", line); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("types.db: cannot parse data source [{0}]", line)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); dataSourceList.Clear(); break; } @@ -150,14 +156,18 @@ public void Load() if (GetDouble(m.Groups[3].Value, out min) != Status.Success) { - Logger.Error("types.db: invalid Min value [{0}]", line); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("types.db: invalid Min value [{0}]", line)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); dataSourceList.Clear(); break; } if (GetDouble(m.Groups[4].Value, out max) != Status.Success) { - Logger.Error("types.db: invalid Max value [{0}]", line); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("types.db: invalid Max value [{0}]", line)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); dataSourceList.Clear(); break; } diff --git a/src/CollectdWinService/Util.cs b/src/CollectdWinService/Util.cs index f625583..4242365 100644 --- a/src/CollectdWinService/Util.cs +++ b/src/CollectdWinService/Util.cs @@ -60,7 +60,6 @@ public static KeyValuePair PostJson(string url, string payload) } else { - Logger.Error("Error posting payload to {0}", url, ex); message = ex.Message; if (ex.Response as HttpWebResponse != null) { @@ -72,6 +71,12 @@ public static KeyValuePair PostJson(string url, string payload) // use a generic client error statusCode = 400; } + + // HTTP return code is used as event log id + LogEventInfo logEvent = LogEventInfo.Create(LogLevel.Error, Logger.Name, String.Format("Error posting payload to {0}", url), ex); + logEvent.Properties.Add("EventID", statusCode); + Logger.Log(logEvent); + } } diff --git a/src/CollectdWinService/WriteAmqpPlugin.cs b/src/CollectdWinService/WriteAmqpPlugin.cs index ec620d8..a765b29 100644 --- a/src/CollectdWinService/WriteAmqpPlugin.cs +++ b/src/CollectdWinService/WriteAmqpPlugin.cs @@ -119,7 +119,10 @@ public void StartConnection() } catch (Exception exp) { - Logger.Error("Got exception when connecting to AMQP broker : ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception when connecting to AMQP broker"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } } } @@ -136,7 +139,10 @@ public void CloseConnection() } catch (Exception exp) { - Logger.Error("Got exception when closing AMQP connection : ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception when closing AMQP connection"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } _connected = false; } diff --git a/src/CollectdWinService/WriteHTTPPlugin.cs b/src/CollectdWinService/WriteHTTPPlugin.cs index c665a1d..a80cbb1 100644 --- a/src/CollectdWinService/WriteHTTPPlugin.cs +++ b/src/CollectdWinService/WriteHTTPPlugin.cs @@ -65,7 +65,10 @@ public void Write(Queue values) } catch (Exception ex) { - Logger.Error("WriteHTTP Failed", ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "WriteHTTP failed"); + logEvent.Exception = ex; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } } } diff --git a/src/CollectdWinService/WriteStatsdPlugin.cs b/src/CollectdWinService/WriteStatsdPlugin.cs index ce0b42f..fce8b3d 100644 --- a/src/CollectdWinService/WriteStatsdPlugin.cs +++ b/src/CollectdWinService/WriteStatsdPlugin.cs @@ -93,7 +93,10 @@ public void Write(Queue values) } catch (Exception ex) { - Logger.Error("WriteStatsd failed", ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "WriteStatsD failed"); + logEvent.Exception = ex; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } } } diff --git a/src/CollectdWinService/app.config b/src/CollectdWinService/app.config index f5a8838..8f64218 100644 --- a/src/CollectdWinService/app.config +++ b/src/CollectdWinService/app.config @@ -41,7 +41,8 @@ + layout="${message}${newline}${exception:format=ToString}" + eventId="${event-properties:EventID}" /> From be549994916bd7e6ef5a188d36c76dc802209647 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Thu, 21 Jul 2016 14:12:31 +0100 Subject: [PATCH 27/30] PHX-5032 sanitise metric FQNs --- src/CollectdWinService/WriteNetuitivePlugin.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/CollectdWinService/WriteNetuitivePlugin.cs b/src/CollectdWinService/WriteNetuitivePlugin.cs index fb784a2..7876591 100644 --- a/src/CollectdWinService/WriteNetuitivePlugin.cs +++ b/src/CollectdWinService/WriteNetuitivePlugin.cs @@ -10,6 +10,7 @@ using System.Runtime.Serialization.Json; using System.IO; using System.Text; +using System.Text.RegularExpressions; namespace Netuitive.CollectdWin { @@ -307,6 +308,7 @@ public void GetIngestMetrics(MetricValue metric, out List metrics, if (metric.TypeInstanceName.Length > 0) metricId += "." + metric.TypeInstanceName; + metricId = Regex.Replace(metricId, "[^a-zA-Z0-9\\._-]", ""); if (metric.Values.Length == 1) { // Simple case - just one metric in type From 23b36a21942bea0e808c8057a1587aa401f865a3 Mon Sep 17 00:00:00 2001 From: Andrew Paine Date: Thu, 21 Jul 2016 17:16:11 +0100 Subject: [PATCH 28/30] PHX-5032 replace spaces with underscores --- src/CollectdWinService/WriteNetuitivePlugin.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CollectdWinService/WriteNetuitivePlugin.cs b/src/CollectdWinService/WriteNetuitivePlugin.cs index 7876591..e399b60 100644 --- a/src/CollectdWinService/WriteNetuitivePlugin.cs +++ b/src/CollectdWinService/WriteNetuitivePlugin.cs @@ -308,7 +308,8 @@ public void GetIngestMetrics(MetricValue metric, out List metrics, if (metric.TypeInstanceName.Length > 0) metricId += "." + metric.TypeInstanceName; - metricId = Regex.Replace(metricId, "[^a-zA-Z0-9\\._-]", ""); + metricId = Regex.Replace(metricId, "[ ]", "_"); // Keep spaces as underscores + metricId = Regex.Replace(metricId, "[^a-zA-Z0-9\\._-]", ""); // Remove punctuation if (metric.Values.Length == 1) { // Simple case - just one metric in type From 7148e7fe1f32cea40640b00deaf8138e687e2496 Mon Sep 17 00:00:00 2001 From: rbellary-vi <73928743+rbellary-vi@users.noreply.github.com> Date: Tue, 9 Feb 2021 17:00:02 +0530 Subject: [PATCH 29/30] Update README.md updating all details --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aaae300..e2a21a3 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ Netuitive Windows Agent ======================== -The Netuitive Windows Agent leverages CollectdWin to collect, aggregate, and publish windows performance counters and attributes to Netuitive. It is designed to expose crucial metrics from your Windows machines and display them in a meaningful way in [Netuitive](https://http://www.netuitive.com/). +The Netuitive Windows Agent leverages CollectdWin to collect, aggregate, and publish windows performance counters and attributes to Netuitive. It is designed to expose crucial metrics from your Windows machines and display them in a meaningful way in [Virtana](https://www.virtana.com/products/cloudwisdom/). -See the [Netuitive Windows agent docs](https://help.netuitive.com/Content/Misc/Datasources/Windows/new_windows_datasource.htm) or the [wiki](../../wiki) for more information, or contact Netuitive support at [support@netuitive.com](mailto:support@netuitive.com). +See the [Virtana Windows agent docs](https://docs.virtana.com/en/windows-agent.html) or the [wiki](../../wiki) for more information, or contact Netuitive support at [cloudwisdom.support@virtana.com](mailto:cloudwisdom.support@virtana.com). Changes to CollectdWin ----------------------- -The base functionality of CollectdWin remains in our fork: exposing windows performance counters for collection and monitoring. The Netuitive Windows Agent diverges from CollectdWin by extending the collection to non-numeric values such as attributes, events, and relationships. Netuitive created plugins to read Windows events and attributes as well as plugins to write to [Netuitive](https://www.netuitive.com/) and [StatsD](https://github.com/etsy/statsd). Netuitive also changed the underlying framework to support collection and representation of elements of different types and metrics from remote sources. +The base functionality of CollectdWin remains in our fork: exposing windows performance counters for collection and monitoring. The Netuitive Windows Agent diverges from CollectdWin by extending the collection to non-numeric values such as attributes, events, and relationships. Netuitive created plugins to read Windows events and attributes as well as plugins to write to [Virtana](https://www.virtana.com/products/cloudwisdom/) and [StatsD](https://github.com/etsy/statsd). Netuitive also changed the underlying framework to support collection and representation of elements of different types and metrics from remote sources. From f3d9af324c48f1c1683972f7f4f3b4d0e33269d0 Mon Sep 17 00:00:00 2001 From: rbellary-vi <73928743+rbellary-vi@users.noreply.github.com> Date: Tue, 9 Feb 2021 21:53:08 +0530 Subject: [PATCH 30/30] Update README.md changed all netuitive to CloudWisdom --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e2a21a3..fd2fa04 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -Netuitive Windows Agent +CloudWisdom Windows Agent ======================== -The Netuitive Windows Agent leverages CollectdWin to collect, aggregate, and publish windows performance counters and attributes to Netuitive. It is designed to expose crucial metrics from your Windows machines and display them in a meaningful way in [Virtana](https://www.virtana.com/products/cloudwisdom/). +The CloudWisdom Windows Agent leverages CollectdWin to collect, aggregate, and publish windows performance counters and attributes to CloudWisdom. It is designed to expose crucial metrics from your Windows machines and display them in a meaningful way in [CloudWisdom](https://www.virtana.com/products/cloudwisdom/). See the [Virtana Windows agent docs](https://docs.virtana.com/en/windows-agent.html) or the [wiki](../../wiki) for more information, or contact Netuitive support at [cloudwisdom.support@virtana.com](mailto:cloudwisdom.support@virtana.com). Changes to CollectdWin ----------------------- -The base functionality of CollectdWin remains in our fork: exposing windows performance counters for collection and monitoring. The Netuitive Windows Agent diverges from CollectdWin by extending the collection to non-numeric values such as attributes, events, and relationships. Netuitive created plugins to read Windows events and attributes as well as plugins to write to [Virtana](https://www.virtana.com/products/cloudwisdom/) and [StatsD](https://github.com/etsy/statsd). Netuitive also changed the underlying framework to support collection and representation of elements of different types and metrics from remote sources. +The base functionality of CollectdWin remains in our fork: exposing windows performance counters for collection and monitoring. The CloudWisdom Windows Agent diverges from CollectdWin by extending the collection to non-numeric values such as attributes, events, and relationships. Netuitive created plugins to read Windows events and attributes as well as plugins to write to [CloudWisdom](https://www.virtana.com/products/cloudwisdom/) and [StatsD](https://github.com/etsy/statsd). Netuitive also changed the underlying framework to support collection and representation of elements of different types and metrics from remote sources.