diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ac75e5c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,70 @@ +Contributing to the Netuitive Windows Agent +========================================== +Contributions are welcome and can be represented in many different ways as noted below. Help is +greatly appreciated and credit will always be given. + + +Types of Contributions +------------------------------ + +### Reporting Bugs +Report bugs on [the issues page](https://github.com/Netuitive/collectdwin/issues). +With your bug report, please include: +- Your operating system name and version. +- Any details about your local setup that might be helpful in troubleshooting the issue. +- Detailed steps to reproduce the bug. + +### Fixing bugs +Find bugs at [the issues page](https://github.com/Netuitive/collectdwin/issues). Anything tagged with +"bug" is open to be fixed. +With your fix, please include: +- The issue number +- A detailed commit message + +### Implementing Features +Find features at [the issues page](https://github.com/Netuitive/collectdwin/issues). Anything tagged +with "feature" is open to be implemented. +With your feature, please include: +- The issue number +- A detailed commit message + +### Writing Documentation +The Netuitive Windows Agent can always use documentation (more documentation is always better!). +Please document your features or usage as part of the official docs/wiki, in docstrings, +in blog posts, articles, or wherever you see fit. + +### Submitting Feedback +File an issue at [the issues page](https://github.com/Netuitive/collectdwin/issues). +If you are proposing a feature: +- Explain how it would work in detail +- Keep the scope as narrow as possible to make it easier to implement + +Workflow +------------------------------ + +1. Create a branch directly in this repo or a fork (if you don't have push access). Please name +branches within this repository `feature/` or `fix/description`. For example, +something like `feature/upgrade_agent_0.2.3-70`. + +1. Create an issue or open a pull request (PR). If you aren't sure your PR will solve the issue +or may be controversial, we're okay with you opening an issue separately and linking to it in +your PR. That way, if the PR is not accepted, the issue will remain and be tracked. + +1. Clone the fork/branch locally. + +1. Close (and reference) issues by the `closes #XXX` or `fixes #XXX` notation in the commit +message. Please use a descriptive, useful commit message that could be used to understand why a +particular change was made. + +1. Keep pushing commits to the initial branch using `--amend`/`--rebase` as necessary. Don't mix +unrelated issues in a single branch. + +1. Clean up the branch (rebase with master to synchronize, squash, edit commits, test, etc.) to +prepare for it to be merged. + +1. If you didn't open a pull request already, do so now. + +1. After reviewing your commits for documentation, passed continuous integration (CI) tests, +version bumps, changelogs, and good, descriptive commit messages, a project maintainer can merge your request. + +1. Create/update the changelog if necessary. \ No newline at end of file diff --git a/README.md b/README.md index 3456096..fd2fa04 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,12 @@ -#Netuitive-CollectdWin +CloudWisdom Windows Agent +======================== -CollectdWin is a MS Windows service which collects, aggregates and publishes windows performance counters and attributes. CollectdWin is similar in concept and design to Collectd (https://collectd.org). +The CloudWisdom Windows Agent leverages CollectdWin to collect, aggregate, and publish windows performance counters and attributes to CloudWisdom. It is designed to expose crucial metrics from your Windows machines and display them in a meaningful way in [CloudWisdom](https://www.virtana.com/products/cloudwisdom/). -This version was forked to add Write Netuitive and other plugins. +See the [Virtana Windows agent docs](https://docs.virtana.com/en/windows-agent.html) or the [wiki](../../wiki) for more information, or contact Netuitive support at [cloudwisdom.support@virtana.com](mailto:cloudwisdom.support@virtana.com). + +Changes to CollectdWin +----------------------- + +The base functionality of CollectdWin remains in our fork: exposing windows performance counters for collection and monitoring. The CloudWisdom Windows Agent diverges from CollectdWin by extending the collection to non-numeric values such as attributes, events, and relationships. Netuitive created plugins to read Windows events and attributes as well as plugins to write to [CloudWisdom](https://www.virtana.com/products/cloudwisdom/) and [StatsD](https://github.com/etsy/statsd). Netuitive also changed the underlying framework to support collection and representation of elements of different types and metrics from remote sources. -For more info, see the [wiki](../../wiki) diff --git a/src/CollectdWinService/CollectableValue.cs b/src/CollectdWinService/CollectableValue.cs index dc7de37..8750f00 100644 --- a/src/CollectdWinService/CollectableValue.cs +++ b/src/CollectdWinService/CollectableValue.cs @@ -137,13 +137,15 @@ internal class EventValue : CollectableValue public long Id { get; set; } public string Message { get; set; } public string Level { get; set; } + public string Title { get; set; } public long Timestamp { get; set; } - private const string JSON_FORMAT = @"{{""level"": ""{0}"", ""source"":""{1}"", ""message"":""{2}"", ""timestamp"": {3} }}"; + private const string JSON_FORMAT = @"{{""level"": ""{0}"", ""source"":""{1}"", ""title"":""{2}"", ""message"":""{3}"", ""timestamp"": {4} }}"; - public EventValue(string hostname, long timestamp, int nLevel, string message, long id) + public EventValue(string hostname, long timestamp, int nLevel, string title, string message, long id) { Level = EventValue.levelToString(nLevel); Timestamp = timestamp; + Title = title; Message = message; HostName = hostname; Id = id; @@ -154,27 +156,6 @@ public EventValue(string hostname, long timestamp, int nLevel, string message, l TypeInstanceName = ""; } - public static int levelToInt(string level) - { - switch (level) - { - case "CRITICAL": - return 1; - case "ERROR": - return 2; - case "WARN": - return 3; - case "WARNING": - return 3; - case "INFO": - return 4; - case "DEBUG": - return 5; - default: // not specified - return -1; - } - } - public static string levelToString(int level) { switch (level) @@ -210,7 +191,7 @@ public override bool Equals(object obj) public override string getJSON() { - return string.Format(JSON_FORMAT, Level, HostName, Message, Timestamp*1000); + return string.Format(JSON_FORMAT, Level, HostName, Title, Message, Timestamp*1000); } } diff --git a/src/CollectdWinService/CollectdWinService.cs b/src/CollectdWinService/CollectdWinService.cs index 1fbd76f..51a0dd1 100644 --- a/src/CollectdWinService/CollectdWinService.cs +++ b/src/CollectdWinService/CollectdWinService.cs @@ -15,6 +15,9 @@ public CollectdWinService() protected override void OnStart(string[] args) { + // Request additional service startup time. Configuring the metrics can take a little while + RequestAdditionalTime(30000); + StartService(args); } diff --git a/src/CollectdWinService/CollectdWinService.csproj b/src/CollectdWinService/CollectdWinService.csproj index 0d5c21a..066a5be 100644 --- a/src/CollectdWinService/CollectdWinService.csproj +++ b/src/CollectdWinService/CollectdWinService.csproj @@ -122,6 +122,7 @@ + diff --git a/src/CollectdWinService/ErrorCodes.cs b/src/CollectdWinService/ErrorCodes.cs new file mode 100644 index 0000000..5574cbc --- /dev/null +++ b/src/CollectdWinService/ErrorCodes.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BloombergFLP.CollectdWin +{ + public static class ErrorCodes + { + public static readonly int ERROR_READ_EXCEEDED_CYCLE_TIME = 1; + public static readonly int ERROR_WRITE_EXCEEDED_CYCLE_TIME = 2; + public static readonly int ERROR_EXCEEDED_MAX_QUEUE_LENGTH = 3; + public static readonly int ERROR_UNHANDLED_EXCEPTION = 4; + public static readonly int ERROR_CONFIGURATION_EXCEPTION = 5; + } +} diff --git a/src/CollectdWinService/MetricsCollector.cs b/src/CollectdWinService/MetricsCollector.cs index 3933dad..2246078 100644 --- a/src/CollectdWinService/MetricsCollector.cs +++ b/src/CollectdWinService/MetricsCollector.cs @@ -28,7 +28,9 @@ public MetricsCollector() var config = ConfigurationManager.GetSection("CollectdWinConfig") as CollectdWinConfig; if (config == null) { - Logger.Error("Cannot get configuration section"); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Cannot get configuration section"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); return; } @@ -140,7 +142,10 @@ private void ReadThreadProc() _collectedValueQueue.Dequeue(); if ((++numMetricsDropped%1000) == 0) { - Logger.Error("Number of metrics dropped : {0}", numMetricsDropped); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exceeded max queue length"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_EXCEEDED_MAX_QUEUE_LENGTH); + Logger.Log(logEvent); + Logger.Warn("Number of metrics dropped : {0}", numMetricsDropped); } } } @@ -151,10 +156,17 @@ private void ReadThreadProc() double revisedInterval = (_interval - elapsed) * 1000; if (revisedInterval / _interval < 0.1) { - Logger.Error("Read thread took {0} seconds out of {1} second cycle", elapsed, _interval); + Logger.Warn("Read thread took {0} seconds out of {1} second cycle", elapsed, _interval); } if (revisedInterval > 0) Thread.Sleep((int)revisedInterval); + else + { + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Read thread exceeded cycle time"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_READ_EXCEEDED_CYCLE_TIME); + Logger.Log(logEvent); + } + } catch (ThreadInterruptedException) { @@ -162,7 +174,10 @@ private void ReadThreadProc() } catch (Exception exp) { - Logger.Error("ReadThreadProc() got exception : ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception in ReadThreadProc()"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); Thread.Sleep(_interval * 1000); } } @@ -216,17 +231,22 @@ private void WriteThreadProc() } } } - double writeEnd = Util.GetNow(); - Logger.Info("Written {0} values in {1:0.00}s", numValues, (writeEnd - writeStart)); - + double writeEnd = Util.GetNow(); double elapsed = writeEnd - writeStart; double revisedInterval = (_interval - elapsed) * 1000; + if (revisedInterval / _interval < 0.1) { - Logger.Error("Write thread took {0} seconds out of {1} second cycle", elapsed, _interval); + Logger.Warn("Write thread took {0} seconds out of {1} second cycle", elapsed, _interval); } - if (revisedInterval > 0) + if (revisedInterval >= 0) Thread.Sleep((int)revisedInterval); + else + { + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Write thread exceeded cycle time"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_WRITE_EXCEEDED_CYCLE_TIME); + Logger.Log(logEvent); + } } catch (ThreadInterruptedException) @@ -235,7 +255,10 @@ private void WriteThreadProc() } catch (Exception exp) { - Logger.Error("WriteThreadProc() got exception : ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception in WriteThreadProc()"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); Thread.Sleep(_interval * 1000); } } @@ -258,7 +281,10 @@ private void AggregatorThreadProc() } catch (Exception exp) { - Logger.Error("AggregatorThreadProc() got exception : ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception in AggregatorThreadProc()"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } } Logger.Trace("AggregatorThreadProc() return"); diff --git a/src/CollectdWinService/PluginRegistry.cs b/src/CollectdWinService/PluginRegistry.cs index 73ecc05..818c865 100644 --- a/src/CollectdWinService/PluginRegistry.cs +++ b/src/CollectdWinService/PluginRegistry.cs @@ -18,7 +18,9 @@ public PluginRegistry() var config = ConfigurationManager.GetSection("CollectdWinConfig") as CollectdWinConfig; if (config == null) { - Logger.Error("Cannot get configuration section"); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Cannot get configuration section"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); return; } foreach (PluginConfig pluginConfig in @@ -37,7 +39,9 @@ public IList CreatePlugins() Type classType = Type.GetType(entry.Value); if (classType == null) { - Logger.Error("Cannot create plugin:{0}, class:{1}", entry.Key, entry.Value); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("Cannot create plugin:{0}, class:{1}", entry.Key, entry.Value)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); continue; } var plugin = (ICollectdPlugin) Activator.CreateInstance(classType); diff --git a/src/CollectdWinService/Properties/AssemblyInfo.cs b/src/CollectdWinService/Properties/AssemblyInfo.cs index 48abb6c..c1a23a2 100644 --- a/src/CollectdWinService/Properties/AssemblyInfo.cs +++ b/src/CollectdWinService/Properties/AssemblyInfo.cs @@ -35,4 +35,4 @@ // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.6.*")] +[assembly: AssemblyVersion("0.7.1.*")] diff --git a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs index b9ea4dd..eb8aada 100644 --- a/src/CollectdWinService/ReadWindowsAttributesPlugin.cs +++ b/src/CollectdWinService/ReadWindowsAttributesPlugin.cs @@ -11,6 +11,7 @@ using System.Runtime.Serialization; using System.Runtime.Serialization.Json; using System.IO; +using System.Net.Sockets; namespace Netuitive.CollectdWin { @@ -26,6 +27,7 @@ internal class ReadWindowsAttributesPlugin : ICollectdReadPlugin private readonly IList _attributes; private string _hostName; private bool _readEC2InstanceMetadata; + private bool _readIPAddress; public ReadWindowsAttributesPlugin() { @@ -42,6 +44,7 @@ public void Configure() _hostName = Util.GetHostName(); _readEC2InstanceMetadata = config.ReadEC2InstanceMetadata; + _readIPAddress = config.ReadIPAddress; _attributes.Clear(); foreach (EnvironmentVariableConfig attr in config.EnvironmentVariables) @@ -92,7 +95,7 @@ public IList Read() } catch (Exception ex) { - Logger.Error(string.Format("Failed to collect attribute: {0}", attribute.variableName), ex); + Logger.Warn(string.Format("Failed to collect attribute: {0}", attribute.variableName), ex); } } return collectedValueList; @@ -150,10 +153,25 @@ private IList GetEC2Metadata() { Logger.Warn("Failed to get EC2 instance metadata. If this server is not an EC2 update the ReadWindowsAttributes.config file to disable collection.", ex); } + catch (Exception ex) + { + Logger.Warn("Failed to process EC2 instance metadata. If this server is not an EC2 update the ReadWindowsAttributes.config file to disable collection.", ex); + } return values; } + private String BytesToString(long numBytes) + { + string[] suffix = { " B", " KB", " MB", " GB", " TB", " PB"}; + if (numBytes == 0) + return "0" + suffix[0]; + long bytes = Math.Abs(numBytes); + int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024))); + double num = Math.Round(bytes / Math.Pow(1024, place), 1); + return (Math.Sign(numBytes) * num).ToString() + suffix[place]; + } + private IList GetCommonAttributes() { // Return standard attributes @@ -187,10 +205,36 @@ private IList GetCommonAttributes() } catch (Exception ex) { - Logger.Error("Failed to get system memory", ex); + Logger.Warn("Failed to get system memory", ex); } - AttributeValue ram = new AttributeValue(_hostName, "ram bytes", totalRAM.ToString()); + AttributeValue ramBytes = new AttributeValue(_hostName, "ram bytes", totalRAM.ToString()); + attributes.Add(ramBytes); + AttributeValue ram = new AttributeValue(_hostName, "ram", BytesToString(totalRAM)); attributes.Add(ram); + + if (_readIPAddress) + { + try + { + var host = Dns.GetHostEntry(Dns.GetHostName()); + List addressList = new List(); + foreach (var ip in host.AddressList) + { + // Only get IP V4 addresses for now + if (ip.AddressFamily == AddressFamily.InterNetwork) + addressList.Add(ip.ToString()); + } + + string ipStr = string.Join(",", addressList.ToArray()); + + AttributeValue ipAttr = new AttributeValue(_hostName, "ip", ipStr); + attributes.Add(ipAttr); + } + catch (Exception ex) + { + Logger.Warn("Failed to get IP address", ex); + } + } return attributes; } diff --git a/src/CollectdWinService/ReadWindowsEventsPlugin.cs b/src/CollectdWinService/ReadWindowsEventsPlugin.cs index 8277cbe..26695ea 100644 --- a/src/CollectdWinService/ReadWindowsEventsPlugin.cs +++ b/src/CollectdWinService/ReadWindowsEventsPlugin.cs @@ -7,14 +7,20 @@ using BloombergFLP.CollectdWin; using System.Text.RegularExpressions; + namespace Netuitive.CollectdWin { internal struct EventQuery { public string log; public string source; + public int minLevel; public int maxLevel; public string filterExp; + public string title; + public int maxPerCycle; + public int minEventId; + public int maxEventId; } internal class ReadWindowsEventsPlugin : ICollectdReadPlugin @@ -23,6 +29,8 @@ internal class ReadWindowsEventsPlugin : ICollectdReadPlugin private readonly IList _events; private string _hostName; private int _interval; + private int _intervalMultiplier; + private long _intervalCounter; public ReadWindowsEventsPlugin() { @@ -40,22 +48,27 @@ public void Configure() _interval = baseConfig.GeneralSettings.Interval; _hostName = Util.GetHostName(); + _intervalMultiplier = config.IntervalMultiplier; + _intervalCounter = 0; _events.Clear(); - foreach (WindowsEventConfig eventConfig in config.Events) { - int level = EventValue.levelToInt(eventConfig.MaxLevel); - EventQuery evt = new EventQuery { log = eventConfig.Log, source = eventConfig.Source, filterExp = eventConfig.FilterExp, - maxLevel = level + minLevel = eventConfig.MinLevel, + maxLevel = eventConfig.MaxLevel, + maxPerCycle = eventConfig.MaxEventsPerCycle, + title = eventConfig.Title, + minEventId = eventConfig.MinEventId, + maxEventId = eventConfig.MaxEventId + }; _events.Add(evt); - Logger.Info("Added event reader: {0}, {1}, {2}", evt.log, evt.source, evt.maxLevel); + Logger.Info("Added event reader {0}: log:{1}, source:{2}, level:{3}-{4}, ID:{5}-{6}, maxAllowed:{7}", evt.title, evt.log, evt.source, evt.minLevel, evt.maxLevel, evt.minEventId, evt.maxEventId, evt.maxPerCycle); } Logger.Info("ReadWindowsEvents plugin configured"); @@ -73,46 +86,106 @@ public void Stop() public IList Read() { + // Only collect events and event metric every nth interval + if (_intervalCounter++ % _intervalMultiplier != 0) + return new List(); + IList collectableValues = new List(); + long totalEvents = 0; + long collectionTime = (long)(Util.toEpoch(DateTime.UtcNow)); + + List recordIds = new List(); foreach (EventQuery eventQuery in _events) { - List records = GetEventRecords(eventQuery.maxLevel, eventQuery.log, eventQuery.source); - foreach (EventRecord record in records) + List records = GetEventRecords(eventQuery.minLevel, eventQuery.maxLevel, eventQuery.log, eventQuery.source); + + // Filter the events - event ID must be in target range, description must match regex and we mustn't have already read this event record ID in another query + Regex filterExp = new Regex(eventQuery.filterExp, RegexOptions.None); + List filteredRecords = records.FindAll(delegate(EventRecord record) + { + return !recordIds.Contains((long)record.RecordId) && record.Id >= eventQuery.minEventId && record.Id <= eventQuery.maxEventId && filterExp.IsMatch(record.FormatDescription()); + }); + + // Add these record IDs to dedupe list so we don't capture them again in a later query + filteredRecords.ForEach(delegate(EventRecord record) { recordIds.Add((long)record.RecordId); }); + + if (filteredRecords.Count <= eventQuery.maxPerCycle) + { + foreach (EventRecord record in filteredRecords) + { + // Timestamp from record is machine time, not GMT + long timestamp = (long)(Util.toEpoch(record.TimeCreated.Value.ToUniversalTime())); + long id = (long)record.RecordId; + string message = record.FormatDescription(); + EventValue newevent = new EventValue(_hostName, timestamp, record.Level.Value, eventQuery.title, message, id); + collectableValues.Add(newevent); + totalEvents++; + } + } + else { - // Timestamp from record is machine time, not GMT - long timestamp = (long)(Util.toEpoch(record.TimeCreated.Value.ToUniversalTime())); - long id = (long)record.RecordId; - string message = record.FormatDescription(); - EventValue newevent = new EventValue(_hostName, timestamp, record.Level.Value, message, id); - - // Dedupe - bool add = true; - foreach(EventValue ev in collectableValues) { - if (ev.Equals(newevent)) + // Too many events - summarise by counting events by application,level and code + Dictionary detailMap = new Dictionary(); + int minLevel = 999; // used to get the most severe event in the period for the summary level + filteredRecords.ForEach(delegate(EventRecord record) + { + string key = string.Format("{0} in {1} ({2})", record.LevelDisplayName, record.ProviderName, record.Id); + + if (record.Level.Value < minLevel) + minLevel = record.Level.Value; + + if (detailMap.ContainsKey(key)) { - add = false; - break; + detailMap[key] = detailMap[key] + 1; } - } + else + { + detailMap.Add(key, 1); + } + }); - // Filter - if (add) + List> detailList = new List>(); + foreach (string key in detailMap.Keys) { - Regex regex = new Regex(eventQuery.filterExp, RegexOptions.None); - add &= regex.IsMatch(message); + detailList.Add(new KeyValuePair(key, detailMap[key])); } + detailList.Sort(delegate(KeyValuePair pair1, KeyValuePair pair2) { return -pair1.Value.CompareTo(pair2.Value); }); - if (add) - collectableValues.Add(newevent); + string[] messageLines = new string[detailList.Count]; + + int ix = 0; + foreach (KeyValuePair pair in detailList) + { + messageLines[ix++] = pair.Value + " x " + pair.Key; + } + string title = string.Format("{0} ({1} events)", eventQuery.title, filteredRecords.Count); + EventValue newevent = new EventValue(_hostName, collectionTime, minLevel, title, String.Join(", ", messageLines), 0); + collectableValues.Add(newevent); + totalEvents += filteredRecords.Count; } - } - + } + + // Add event count metric + MetricValue eventCountMetric = new MetricValue + { + HostName = _hostName, + PluginName = "windows_events", + PluginInstanceName = "", + TypeName = "count", + TypeInstanceName = "event_count", + Values = new double[] { totalEvents }, + FriendlyNames = new string[] { "Windows Event Count" }, + Epoch = collectionTime + }; + collectableValues.Add(eventCountMetric); + return collectableValues; } // 1 = critical, 2=error, 3=warning, 4=information,5=verbose, -1=no filter - private List GetEventRecords(int level, string logName,string providerName) { + private List GetEventRecords(int minLevel, int maxLevel, string logName,string providerName) { List eventRecords = new List(); + long eventInterval = _intervalMultiplier * _interval * 1000; EventRecord eventRecord; string queryString; @@ -120,19 +193,11 @@ private List GetEventRecords(int level, string logName,string provi { if (providerName != null && providerName.Length > 0) { - queryString = "*"; - if (level >= 0) - queryString += String.Format("[System/Level <= {0}]", level); - - queryString += String.Format("[System/Provider/@Name = '{0}'][System/TimeCreated[timediff(@SystemTime) <= {1}]]", providerName, _interval * 1000); + queryString = String.Format("*[System[(Level >= {0}) and (Level <= {1}) and Provider/@Name = '{2}' and TimeCreated[timediff(@SystemTime) <= {3}]]]", minLevel, maxLevel, providerName, eventInterval); } else { - queryString = "*[System["; - if (level >= 0) - queryString += String.Format("(Level <= {0}) and ", level); - queryString += String.Format( - "TimeCreated[timediff(@SystemTime) <= {0}]]]", _interval * 1000); + queryString = String.Format("*[System[(Level >= {0}) and (Level <= {1}) and TimeCreated[timediff(@SystemTime) <= {2}]]]", minLevel, maxLevel, eventInterval); } EventLogQuery query = new EventLogQuery(logName, PathType.LogName, queryString); EventLogReader reader = new EventLogReader(query); @@ -151,7 +216,10 @@ private List GetEventRecords(int level, string logName,string provi } catch (Exception ex) { - Logger.Error(ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Unhandled Exception in ReadWindowsEventsPlugin"); + logEvent.Exception = ex; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } return eventRecords; diff --git a/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs b/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs index 43727af..245fb2e 100644 --- a/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs +++ b/src/CollectdWinService/ReadWindowsPerfCountersPlugin.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using NLog; using System.Text.RegularExpressions; +using System.Threading; namespace BloombergFLP.CollectdWin { @@ -91,18 +92,30 @@ public void Configure() } catch (ArgumentException ex) { - Logger.Error(string.Format("Failed to parse instance regular expression: category={0}, instance={1}, counter={2}", counter.Category, counter.Instance, counter.Name), ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); + Logger.Warn(string.Format("Failed to parse instance regular expression: category={0}, instance={1}, counter={2}", counter.Category, counter.Instance, counter.Name), ex); } catch (InvalidOperationException ex) { if (ex.Message.ToLower().Contains("category does not exist")) { Logger.Warn(string.Format("Performance Counter not added: Category does not exist: {0}", counter.Category)); - } else - Logger.Error(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", counter.Category, counter.Instance, counter.Name), ex); + } + else + { + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); + Logger.Warn(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", counter.Category, counter.Instance, counter.Name), ex); + } } catch (Exception ex) { - Logger.Error(string.Format("Could not initialise performance counter category: {0}", counter.Category), ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Failed to initialise performance counter"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); + Logger.Warn(string.Format("Could not initialise performance counter category: {0}", counter.Category), ex); } if (instances.Length == 0) { @@ -137,12 +150,15 @@ public void Configure() } } } + // Wait 1 second for the two-valued counters to be ready for next incremental read - see https://msdn.microsoft.com/en-us/library/system.diagnostics.performancecounter.nextvalue(v=vs.110).aspx + Thread.Sleep(1000); + Logger.Info("ReadWindowsPerfeCounters plugin configured {0} metrics", metricCounter); } public void Start() { - Logger.Info("ReadWindowsPerfCountesr plugin started"); + Logger.Info("ReadWindowsPerfCounters plugin started"); } public void Stop() @@ -193,7 +209,7 @@ public IList Read() } catch (Exception ex) { - Logger.Error(string.Format("Failed to collect metric: {0}, {1}, {2}", metric.Category, metric.Instance, metric.CounterName), ex); + Logger.Warn(string.Format("Failed to collect metric: {0}, {1}, {2}", metric.Category, metric.Instance, metric.CounterName), ex); } } return (metricValueList); @@ -218,7 +234,10 @@ private Boolean AddPerformanceCounter(string category, string names, string inst int ix = 0; foreach (string ctr in counterList) { - metric.Counters.Add(new PerformanceCounter(category, ctr.Trim(), instance)); + PerformanceCounter perfCounter = new PerformanceCounter(category, ctr.Trim(), instance); + // Collect a value - this is needed to initialise counters that need two values + perfCounter.NextValue(); + metric.Counters.Add(perfCounter); string friendlyName = ctr.Trim(); if (instance.Length > 0) friendlyName += " (" + instance + ")"; @@ -242,13 +261,23 @@ private Boolean AddPerformanceCounter(string category, string names, string inst { if (ex.Message.ToLower().Contains("category does not exist")) { Logger.Warn(string.Format("Performance Counter not added: Category does not exist: {0}", category)); - } else - Logger.Error(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", category, instance, names), ex); + } + else + { + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Could not initialise performance counter"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); + Logger.Warn(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", category, instance, names), ex); + return false; + } return false; } catch (Exception ex) { - Logger.Error(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", category, instance, names), ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Could not initialise performance counter"); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); + Logger.Warn(string.Format("Could not initialise performance counter category: {0}, instance: {1}, counter: {2}", category, instance, names), ex); return false; } } diff --git a/src/CollectdWinService/StatsdListener.cs b/src/CollectdWinService/StatsdListener.cs index 9057d90..11baadb 100644 --- a/src/CollectdWinService/StatsdListener.cs +++ b/src/CollectdWinService/StatsdListener.cs @@ -36,7 +36,10 @@ private void BindSocket() } catch (Exception exp) { - Logger.Error("BindSocket failed: ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "BindSocket failed"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } if (_socket.IsBound) break; @@ -53,7 +56,10 @@ private void CloseSocket() } catch (Exception exp) { - Logger.Error("CloseSocket failed: ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "CloseSocket failed"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } } diff --git a/src/CollectdWinService/TypesDB.cs b/src/CollectdWinService/TypesDB.cs index e550013..77e058b 100644 --- a/src/CollectdWinService/TypesDB.cs +++ b/src/CollectdWinService/TypesDB.cs @@ -123,14 +123,18 @@ public void Load() Match match = dataSetRegex.Match(line); if (match.Groups.Count < 3) { - Logger.Error("types.db: invalid data set [{0}]", line); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("types.db: invalid data set [{0}]", line)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); continue; } string dataSetName = match.Groups[1].Value; MatchCollection matches = dataSourceRegex.Matches(line); if (matches.Count < 1) { - Logger.Error("types.db: invalid data source [{0}]", line); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("types.db: invalid data source [{0}]", line)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); continue; } var dataSourceList = new List(); @@ -138,7 +142,9 @@ public void Load() { if (m.Groups.Count != 5) { - Logger.Error("types.db: cannot parse data source [{0}]", line); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("types.db: cannot parse data source [{0}]", line)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); dataSourceList.Clear(); break; } @@ -150,14 +156,18 @@ public void Load() if (GetDouble(m.Groups[3].Value, out min) != Status.Success) { - Logger.Error("types.db: invalid Min value [{0}]", line); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("types.db: invalid Min value [{0}]", line)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); dataSourceList.Clear(); break; } if (GetDouble(m.Groups[4].Value, out max) != Status.Success) { - Logger.Error("types.db: invalid Max value [{0}]", line); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, String.Format("types.db: invalid Max value [{0}]", line)); + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_CONFIGURATION_EXCEPTION); + Logger.Log(logEvent); dataSourceList.Clear(); break; } diff --git a/src/CollectdWinService/Util.cs b/src/CollectdWinService/Util.cs index c726f60..4242365 100644 --- a/src/CollectdWinService/Util.cs +++ b/src/CollectdWinService/Util.cs @@ -3,6 +3,7 @@ using System.Net; using System.Threading; using NLog; +using System.Collections.Generic; namespace BloombergFLP.CollectdWin { @@ -37,18 +38,17 @@ public static string GetHostName() return (Environment.MachineName.ToLower()); } - public static string PostJson(string url, string payload) + public static KeyValuePair PostJson(string url, string payload) { -// Logger.Debug("WriteNetuitive: {0}", payload); -// Uri uri = new Uri("http://127.0.0.1:8888"); - string result = ""; + string message = ""; + int statusCode = 200; + try { - using (var client = new WebClient()) { client.Headers[HttpRequestHeader.ContentType] = "application/json"; - result = client.UploadString(url, "POST", payload); + message = client.UploadString(url, "POST", payload); } } catch (System.Net.WebException ex) @@ -60,13 +60,27 @@ public static string PostJson(string url, string payload) } else { - Logger.Error("Error posting payload to {0}", url, ex); - return ex.Message; + message = ex.Message; + if (ex.Response as HttpWebResponse != null) + { + // Get the actual code + statusCode = ((HttpWebResponse)ex.Response).StatusCode.GetHashCode(); + } + else + { + // use a generic client error + statusCode = 400; + } + + // HTTP return code is used as event log id + LogEventInfo logEvent = LogEventInfo.Create(LogLevel.Error, Logger.Name, String.Format("Error posting payload to {0}", url), ex); + logEvent.Properties.Add("EventID", statusCode); + Logger.Log(logEvent); + } } - return result; - + return new KeyValuePair(statusCode, message); } } } diff --git a/src/CollectdWinService/WriteAmqpPlugin.cs b/src/CollectdWinService/WriteAmqpPlugin.cs index ec620d8..a765b29 100644 --- a/src/CollectdWinService/WriteAmqpPlugin.cs +++ b/src/CollectdWinService/WriteAmqpPlugin.cs @@ -119,7 +119,10 @@ public void StartConnection() } catch (Exception exp) { - Logger.Error("Got exception when connecting to AMQP broker : ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception when connecting to AMQP broker"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } } } @@ -136,7 +139,10 @@ public void CloseConnection() } catch (Exception exp) { - Logger.Error("Got exception when closing AMQP connection : ", exp); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "Exception when closing AMQP connection"); + logEvent.Exception = exp; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } _connected = false; } diff --git a/src/CollectdWinService/WriteHTTPPlugin.cs b/src/CollectdWinService/WriteHTTPPlugin.cs index c665a1d..a80cbb1 100644 --- a/src/CollectdWinService/WriteHTTPPlugin.cs +++ b/src/CollectdWinService/WriteHTTPPlugin.cs @@ -65,7 +65,10 @@ public void Write(Queue values) } catch (Exception ex) { - Logger.Error("WriteHTTP Failed", ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "WriteHTTP failed"); + logEvent.Exception = ex; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } } } diff --git a/src/CollectdWinService/WriteNetuitivePlugin.cs b/src/CollectdWinService/WriteNetuitivePlugin.cs index b9f81b3..e399b60 100644 --- a/src/CollectdWinService/WriteNetuitivePlugin.cs +++ b/src/CollectdWinService/WriteNetuitivePlugin.cs @@ -10,6 +10,7 @@ using System.Runtime.Serialization.Json; using System.IO; using System.Text; +using System.Text.RegularExpressions; namespace Netuitive.CollectdWin { @@ -23,6 +24,7 @@ internal class WriteNetuitivePlugin : ICollectdWritePlugin private string _location; private string _defaultElementType; private int _payloadSize; + private bool _enabled; public void Configure() { @@ -57,8 +59,9 @@ public void Configure() Logger.Info("Maximum payload size: {0}", _payloadSize); - _maxEventTitleLength = config.MaxEventTitleLength; + + _enabled = true; } public void Start() @@ -73,6 +76,9 @@ public void Stop() public void Write(CollectableValue value) { + if (!_enabled) + return; + Queue entry = new Queue(); entry.Enqueue(value); Write(entry); @@ -80,6 +86,10 @@ public void Write(CollectableValue value) public void Write(Queue values) { + if (!_enabled) + return; + + double writeStart = Util.GetNow(); // Split into separate lists for each ingest point List metricsAttributesAndRelations = null; @@ -99,8 +109,11 @@ public void Write(Queue values) List eventList = ConvertEventsToIngestEvents(events); // Send event payloads - PostEvents(eventList); + if (eventList.Count > 0) + PostEvents(eventList); + double writeEnd = Util.GetNow(); + Logger.Info("Write took {0:0.00}s", (writeEnd - writeStart)); } protected List ConvertMetricsAttributesAndRelationsToIngestElements(List metricsAttributes) @@ -147,7 +160,7 @@ protected List ConvertEventsToIngestEvents(List events) // Format title and message string message = value.Message; - string title = value.Level + " - " + value.Message; + string title = value.Title; if (title.Length > _maxEventTitleLength) title = title.Substring(0, _maxEventTitleLength); @@ -218,12 +231,14 @@ private void PostEvents(List eventList) { eventPayloads.Add(SerialiseJsonObject(ingestEvent, typeof(IngestEvent))); } - + string eventPayload = "[" + string.Join(",", eventPayloads.ToArray()) + "]"; - string res = Util.PostJson(_eventIngestUrl, eventPayload); - if (res.Length > 0) + KeyValuePair res = Util.PostJson(_eventIngestUrl, eventPayload); + + bool isOK = ProcessResponseCode(res.Key); + if (!isOK) { - Logger.Warn("Error posting events: {0}", res); + Logger.Warn("Error posting events: {0}, {1}", res.Key, res.Value); Logger.Warn("Payload: {0}", eventPayload); } } @@ -234,10 +249,11 @@ private void PostMetricsAndAttributes(List mergedIngestElementLis foreach (IngestElement ingestElement in mergedIngestElementList) { string payload = "[" + SerialiseJsonObject(ingestElement, typeof(IngestElement)) + "]"; - string res = Util.PostJson(_ingestUrl, payload); - if (res.Length > 0) + KeyValuePair res = Util.PostJson(_ingestUrl, payload); + bool isOK = ProcessResponseCode(res.Key); + if (!isOK) { - Logger.Warn("Error posting metrics/attributes: {0}", res); + Logger.Warn("Error posting metrics/attributes: {0}, {1}", res.Key, res.Value); Logger.Warn("Payload: {0}", payload); } } @@ -292,6 +308,8 @@ public void GetIngestMetrics(MetricValue metric, out List metrics, if (metric.TypeInstanceName.Length > 0) metricId += "." + metric.TypeInstanceName; + metricId = Regex.Replace(metricId, "[ ]", "_"); // Keep spaces as underscores + metricId = Regex.Replace(metricId, "[^a-zA-Z0-9\\._-]", ""); // Remove punctuation if (metric.Values.Length == 1) { // Simple case - just one metric in type @@ -331,6 +349,19 @@ protected void GetIngestRelations(RelationValue value, out List relations = new List(); relations.Add(new IngestRelation(value.Fqn)); } + + protected bool ProcessResponseCode(int responseCode) + { + if (responseCode == 410) + { + // shutdown this plugin + Logger.Fatal("Received plugin shutdown code from server"); + _enabled = false; + return false; + } + else return + responseCode >=200 && responseCode < 300; + } } // ******************** DataContract objects for JSON serialisation ******************** @@ -494,6 +525,7 @@ public IngestEvent(string type, string source, string title, long timestamp) this.source = source; this.title = title; this.timestamp = timestamp; + this.tags = new List(); } public void setData(IngestEventData data) diff --git a/src/CollectdWinService/WriteStatsdPlugin.cs b/src/CollectdWinService/WriteStatsdPlugin.cs index ce0b42f..fce8b3d 100644 --- a/src/CollectdWinService/WriteStatsdPlugin.cs +++ b/src/CollectdWinService/WriteStatsdPlugin.cs @@ -93,7 +93,10 @@ public void Write(Queue values) } catch (Exception ex) { - Logger.Error("WriteStatsd failed", ex); + LogEventInfo logEvent = new LogEventInfo(LogLevel.Error, Logger.Name, "WriteStatsD failed"); + logEvent.Exception = ex; + logEvent.Properties.Add("EventID", ErrorCodes.ERROR_UNHANDLED_EXCEPTION); + Logger.Log(logEvent); } } } diff --git a/src/CollectdWinService/app.config b/src/CollectdWinService/app.config index f5a8838..8f64218 100644 --- a/src/CollectdWinService/app.config +++ b/src/CollectdWinService/app.config @@ -41,7 +41,8 @@ + layout="${message}${newline}${exception:format=ToString}" + eventId="${event-properties:EventID}" /> diff --git a/src/CollectdWinService/config/ReadWindowsAttributesPluginConfig.cs b/src/CollectdWinService/config/ReadWindowsAttributesPluginConfig.cs index 5b5a323..c21a2f6 100644 --- a/src/CollectdWinService/config/ReadWindowsAttributesPluginConfig.cs +++ b/src/CollectdWinService/config/ReadWindowsAttributesPluginConfig.cs @@ -19,6 +19,14 @@ public Boolean ReadEC2InstanceMetadata get { return (Boolean)base["ReadEC2InstanceMetadata"]; } set { base["ReadEC2InstanceMetadata"] = value; } } + + [ConfigurationProperty("ReadIPAddress", IsRequired = false, DefaultValue = true)] + public Boolean ReadIPAddress + { + get { return (Boolean)base["ReadIPAddress"]; } + set { base["ReadIPAddress"] = value; } + } + } public class WindowsEnvironmentVariableCollection : ConfigurationElementCollection diff --git a/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs b/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs index 869de94..ebb1d8b 100644 --- a/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs +++ b/src/CollectdWinService/config/ReadWindowsEventPluginConfig.cs @@ -12,6 +12,13 @@ public WindowsEventCollection Events get { return (WindowsEventCollection)base["Events"]; } set { base["Events"] = value; } } + + [ConfigurationProperty("IntervalMultiplier", IsRequired = false, DefaultValue = 5)] + public int IntervalMultiplier + { + get { return (int)base["IntervalMultiplier"]; } + set { base["IntervalMultiplier"] = value; } + } } public sealed class WindowsEventCollection : ConfigurationElementCollection @@ -45,19 +52,54 @@ public String Source set { base["Source"] = value; } } - [ConfigurationProperty("MaxLevel", IsRequired = false, DefaultValue="")] - public string MaxLevel + [ConfigurationProperty("MaxLevel", IsRequired = true)] + public int MaxLevel { - get { return (string)base["MaxLevel"]; } + get { return (int)base["MaxLevel"]; } set { base["MaxLevel"] = value; } } + [ConfigurationProperty("MinLevel", IsRequired = false, DefaultValue = 1)] + public int MinLevel + { + get { return (int)base["MinLevel"]; } + set { base["MinLevel"] = value; } + } + [ConfigurationProperty("FilterExp", IsRequired = false, DefaultValue=".*")] public string FilterExp { get { return (string)base["FilterExp"]; } set { base["FilterExp"] = value; } } + + [ConfigurationProperty("Title", IsRequired = true)] + public string Title + { + get { return (string)base["Title"]; } + set { base["Title"] = value; } + } + + [ConfigurationProperty("MaxEventsPerCycle", IsRequired = false, DefaultValue=1)] + public int MaxEventsPerCycle + { + get { return (int)base["MaxEventsPerCycle"]; } + set { base["MaxEventsPerCycle"] = value; } + } + + [ConfigurationProperty("MinEventId", IsRequired = false, DefaultValue = 0)] + public int MinEventId + { + get { return (int)base["MinEventId"]; } + set { base["MinEventId"] = value; } + } + + [ConfigurationProperty("MaxEventId", IsRequired = false, DefaultValue = 65535)] + public int MaxEventId + { + get { return (int)base["MaxEventId"]; } + set { base["MaxEventId"] = value; } + } } } diff --git a/src/CollectdWinService/config/ReadWindowsEvents.config b/src/CollectdWinService/config/ReadWindowsEvents.config index 91d8d4d..8495b4d 100644 --- a/src/CollectdWinService/config/ReadWindowsEvents.config +++ b/src/CollectdWinService/config/ReadWindowsEvents.config @@ -1,11 +1,23 @@  - + + - + - - + + + + + + \ No newline at end of file diff --git a/src/CollectdWinService/config/ReadWindowsPerfCounters.config b/src/CollectdWinService/config/ReadWindowsPerfCounters.config index 63f871e..7897748 100644 --- a/src/CollectdWinService/config/ReadWindowsPerfCounters.config +++ b/src/CollectdWinService/config/ReadWindowsPerfCounters.config @@ -55,7 +55,7 @@ - - - + CollectdPluginInstance="" CollectdType="count" CollectdTypeInstance="user_connections" /> - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file