From ac5e65416d4308402fe09b6ef2608d221f0c2302 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Sat, 7 Jun 2025 01:41:08 +0900 Subject: [PATCH 01/10] chore: upgrade projects to net9 --- SnaffCore/SnaffCore.csproj | 109 +++---------------------- SnaffCore/UltraSnaffCore.csproj | 118 ++++----------------------- Snaffler/Snaffler.csproj | 137 +++++--------------------------- 3 files changed, 47 insertions(+), 317 deletions(-) diff --git a/SnaffCore/SnaffCore.csproj b/SnaffCore/SnaffCore.csproj index fa0fc804..2c86f732 100644 --- a/SnaffCore/SnaffCore.csproj +++ b/SnaffCore/SnaffCore.csproj @@ -1,98 +1,11 @@ - - - - - Debug - AnyCPU - {B118802D-2E46-4E41-AAC7-9EE890268F8B} - Library - Properties - SnaffCore - SnaffCore - v4.5.1 - 512 - true - - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - false - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - false - - - true - bin\x86\Debug\ - DEBUG;TRACE - full - x86 - 7.3 - prompt - - - bin\x86\Release\ - TRACE - true - pdbonly - x86 - 7.3 - prompt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0.15.0 - - - - \ No newline at end of file + + + net9.0 + false + + + + + + + diff --git a/SnaffCore/UltraSnaffCore.csproj b/SnaffCore/UltraSnaffCore.csproj index 539d3311..fa38f498 100644 --- a/SnaffCore/UltraSnaffCore.csproj +++ b/SnaffCore/UltraSnaffCore.csproj @@ -1,104 +1,14 @@ - - - - - Debug - AnyCPU - {B118802D-2E46-4E41-AAC7-9EE890268F8B} - Library - Properties - SnaffCore - SnaffCore - v4.5.1 - 512 - true - - - - true - full - false - bin\Debug\ - DEBUG;TRACE;ULTRASNAFFLER - prompt - 4 - false - - - pdbonly - true - bin\Release\ - TRACE;ULTRASNAFFLER - prompt - 4 - false - - - true - bin\x86\Debug\ - DEBUG;TRACE;ULTRASNAFFLER - full - x86 - 7.3 - prompt - - - bin\x86\Release\ - TRACE;ULTRASNAFFLER - true - pdbonly - x86 - 7.3 - prompt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0.15.0 - - - 0.24.0 - - - 1.6.1.1 - - - - \ No newline at end of file + + + net9.0 + false + ULTRASNAFFLER + + + + + + + + + diff --git a/Snaffler/Snaffler.csproj b/Snaffler/Snaffler.csproj index f84fdba9..db626596 100644 --- a/Snaffler/Snaffler.csproj +++ b/Snaffler/Snaffler.csproj @@ -1,115 +1,22 @@ - - - - - Debug - AnyCPU - {2AA060B4-DE88-4D2A-A26A-760C1CEFEC3E} - Exe - Snaffler - Snaffler - v4.5.1 - 512 - true - - - - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - false - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - false - - - - - - - true - bin\x86\Debug\ - DEBUG;TRACE - full - x86 - 7.3 - prompt - - - bin\x86\Release\ - TRACE - true - pdbonly - x86 - 7.3 - prompt - - - - - - - - - - - - True - True - Resources.resx - - - - - - - - - {b118802d-2e46-4e41-aac7-9ee890268f8b} - SnaffCore - - - - - - - - - - PublicResXFileCodeGenerator - Resources.Designer.cs - - - - - 3.0.22 - - - 0.5.15 - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - 0.15.0 - - - 4.7.15 - - - - - \ No newline at end of file + + + Exe + net9.0 + false + Snaffler + Snaffler + + + + + + + + + + + + + + + From e9a315f98c26436521167b63d812f0c11a5358da Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Sat, 7 Jun 2025 02:26:15 +0900 Subject: [PATCH 02/10] Add async queue handling --- SnaffCore/Concurrency/BlockingMq.cs | 32 ++++++++++++++++++----------- Snaffler/SnaffleRunner.cs | 23 ++++++++------------- Snaffler/Snaffler.cs | 5 +++-- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/SnaffCore/Concurrency/BlockingMq.cs b/SnaffCore/Concurrency/BlockingMq.cs index 3850de15..c47cb89b 100644 --- a/SnaffCore/Concurrency/BlockingMq.cs +++ b/SnaffCore/Concurrency/BlockingMq.cs @@ -1,6 +1,7 @@ using SnaffCore.Classifiers; using System; using System.Collections.Concurrent; +using System.Threading.Channels; namespace SnaffCore.Concurrency { @@ -19,29 +20,35 @@ public static BlockingMq GetMq() } // Message Queue - public BlockingCollection Q { get; private set; } + private Channel _channel; + public ChannelReader Reader => _channel.Reader; private BlockingMq() { - Q = new BlockingCollection(); + _channel = Channel.CreateUnbounded(); + } + + private void Enqueue(SnafflerMessage message) + { + _channel.Writer.TryWrite(message); } public void Terminate() { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.Fatal, Message = "Terminate was called" }); - //this.Q.CompleteAdding(); + _channel.Writer.TryComplete(); } public void Trace(string message) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.Trace, @@ -52,7 +59,7 @@ public void Trace(string message) public void Degub(string message) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.Degub, @@ -63,7 +70,7 @@ public void Degub(string message) public void Info(string message) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.Info, @@ -73,7 +80,7 @@ public void Info(string message) public void Error(string message) { - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.Error, @@ -84,7 +91,7 @@ public void Error(string message) public void FileResult(FileResult fileResult) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.FileResult, @@ -94,7 +101,7 @@ public void FileResult(FileResult fileResult) public void DirResult(DirResult dirResult) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, DirResult = dirResult, @@ -104,7 +111,7 @@ public void DirResult(DirResult dirResult) public void ShareResult(ShareResult shareResult) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, ShareResult = shareResult, @@ -114,11 +121,12 @@ public void ShareResult(ShareResult shareResult) public void Finish() { - Q.Add(new SnafflerMessage() + Enqueue(new SnafflerMessage() { DateTime = DateTime.Now, Type = SnafflerMessageType.Finish }); + _channel.Writer.TryComplete(); } } } \ No newline at end of file diff --git a/Snaffler/SnaffleRunner.cs b/Snaffler/SnaffleRunner.cs index 4d01882d..2dcd8dbe 100644 --- a/Snaffler/SnaffleRunner.cs +++ b/Snaffler/SnaffleRunner.cs @@ -37,7 +37,7 @@ private string hostString() return _hostString; } - public void Run(string[] args) + public async Task RunAsync(string[] args) { // prime the hoststring lazy instantiator hostString(); @@ -201,16 +201,9 @@ public void Run(string[] args) var tokenSource = new CancellationTokenSource(); var token = tokenSource.Token; - Task thing = Task.Factory.StartNew(() => { controller.Execute(); }, token); - bool exit = false; - - while (exit == false) - { - if (HandleOutput() == true) - { - exit = true; - } - } + Task controllerTask = Task.Run(() => controller.Execute(), token); + await HandleOutputAsync(); + await controllerTask; return; } catch (Exception e) @@ -223,7 +216,7 @@ public void Run(string[] args) private void DumpQueue() { BlockingMq Mq = BlockingMq.GetMq(); - while (Mq.Q.TryTake(out SnafflerMessage message)) + while (Mq.Reader.TryRead(out SnafflerMessage message)) { // emergency dump of queue contents to console Console.WriteLine(message.Message); @@ -234,10 +227,10 @@ private void DumpQueue() } } - private bool HandleOutput() + private async Task HandleOutputAsync() { BlockingMq Mq = BlockingMq.GetMq(); - foreach (SnafflerMessage message in Mq.Q.GetConsumingEnumerable()) + await foreach (SnafflerMessage message in Mq.Reader.ReadAllAsync()) { if (Options.LogType == LogType.Plain) { @@ -248,7 +241,7 @@ private bool HandleOutput() ProcessMessageJSON(message); } - // catch terminating messages and bail out of the master 'while' loop + // catch terminating messages and bail out if ((message.Type == SnafflerMessageType.Fatal) || (message.Type == SnafflerMessageType.Finish)) { return true; diff --git a/Snaffler/Snaffler.cs b/Snaffler/Snaffler.cs index dff04469..6db253e4 100644 --- a/Snaffler/Snaffler.cs +++ b/Snaffler/Snaffler.cs @@ -1,13 +1,14 @@ using System; +using System.Threading.Tasks; namespace Snaffler { public static class Snaffler { - public static void Main(string[] args) + public static async Task Main(string[] args) { SnaffleRunner runner = new SnaffleRunner(); - runner.Run(args); + await runner.RunAsync(args); Console.WriteLine("I snaffled 'til the snafflin was done."); } } From 56da45fa72e45c3e0663c86c55888e8bbabb7625 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Sat, 7 Jun 2025 11:21:35 +0900 Subject: [PATCH 03/10] Add async file IO and DNS operations --- SnaffCore/Concurrency/BlockingMq.cs | 32 +++-- .../Concurrency/BlockingTaskScheduler.cs | 20 +++ SnaffCore/SnaffCon.cs | 21 +-- Snaffler/Config.cs | 122 +++++++++--------- Snaffler/SnaffleRunner.cs | 54 ++++---- Snaffler/Snaffler.cs | 5 +- 6 files changed, 138 insertions(+), 116 deletions(-) diff --git a/SnaffCore/Concurrency/BlockingMq.cs b/SnaffCore/Concurrency/BlockingMq.cs index 3850de15..c47cb89b 100644 --- a/SnaffCore/Concurrency/BlockingMq.cs +++ b/SnaffCore/Concurrency/BlockingMq.cs @@ -1,6 +1,7 @@ using SnaffCore.Classifiers; using System; using System.Collections.Concurrent; +using System.Threading.Channels; namespace SnaffCore.Concurrency { @@ -19,29 +20,35 @@ public static BlockingMq GetMq() } // Message Queue - public BlockingCollection Q { get; private set; } + private Channel _channel; + public ChannelReader Reader => _channel.Reader; private BlockingMq() { - Q = new BlockingCollection(); + _channel = Channel.CreateUnbounded(); + } + + private void Enqueue(SnafflerMessage message) + { + _channel.Writer.TryWrite(message); } public void Terminate() { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.Fatal, Message = "Terminate was called" }); - //this.Q.CompleteAdding(); + _channel.Writer.TryComplete(); } public void Trace(string message) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.Trace, @@ -52,7 +59,7 @@ public void Trace(string message) public void Degub(string message) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.Degub, @@ -63,7 +70,7 @@ public void Degub(string message) public void Info(string message) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.Info, @@ -73,7 +80,7 @@ public void Info(string message) public void Error(string message) { - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.Error, @@ -84,7 +91,7 @@ public void Error(string message) public void FileResult(FileResult fileResult) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, Type = SnafflerMessageType.FileResult, @@ -94,7 +101,7 @@ public void FileResult(FileResult fileResult) public void DirResult(DirResult dirResult) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, DirResult = dirResult, @@ -104,7 +111,7 @@ public void DirResult(DirResult dirResult) public void ShareResult(ShareResult shareResult) { // say we did a thing - Q.Add(new SnafflerMessage + Enqueue(new SnafflerMessage { DateTime = DateTime.Now, ShareResult = shareResult, @@ -114,11 +121,12 @@ public void ShareResult(ShareResult shareResult) public void Finish() { - Q.Add(new SnafflerMessage() + Enqueue(new SnafflerMessage() { DateTime = DateTime.Now, Type = SnafflerMessageType.Finish }); + _channel.Writer.TryComplete(); } } } \ No newline at end of file diff --git a/SnaffCore/Concurrency/BlockingTaskScheduler.cs b/SnaffCore/Concurrency/BlockingTaskScheduler.cs index f748bd3a..0a5b5faa 100644 --- a/SnaffCore/Concurrency/BlockingTaskScheduler.cs +++ b/SnaffCore/Concurrency/BlockingTaskScheduler.cs @@ -66,6 +66,26 @@ public void New(Action action) } } } + + public void New(Func asyncAction) + { + bool proceed = false; + + while (proceed == false) + { + lock (syncLock) + { + if (_maxBacklog != 0) + { + if (Scheduler.GetTaskCounters().CurrentTasksQueued >= _maxBacklog) + continue; + } + + proceed = true; + _taskFactory.StartNew(asyncAction, _cancellationSource.Token).Unwrap(); + } + } + } } public class TaskCounters diff --git a/SnaffCore/SnaffCon.cs b/SnaffCore/SnaffCon.cs index 5334939d..8886344b 100644 --- a/SnaffCore/SnaffCon.cs +++ b/SnaffCore/SnaffCon.cs @@ -16,6 +16,7 @@ using static SnaffCore.Config.Options; using Timer = System.Timers.Timer; using System.Net; +using System.Threading.Tasks; namespace SnaffCore { @@ -80,7 +81,7 @@ public static BlockingStaticTaskScheduler GetFileTaskScheduler() return FileTaskScheduler; } - public void Execute() + public async Task ExecuteAsync() { StartTime = DateTime.Now; // This is the main execution thread. @@ -116,7 +117,7 @@ public void Execute() if (!MyOptions.DfsOnly) { Mq.Info("Invoking full domain computer discovery."); - DomainTargetDiscovery(); + await DomainTargetDiscoveryAsync(); } else { @@ -140,7 +141,7 @@ public void Execute() // or we've been told what computers to hit... else if (MyOptions.ComputerTargets != null) { - ShareDiscovery(MyOptions.ComputerTargets); + await ShareDiscoveryAsync(MyOptions.ComputerTargets); } // but if that hasn't been done, something has gone wrong. @@ -176,7 +177,7 @@ private void DomainDfsDiscovery() } } - private void DomainTargetDiscovery() + private async Task DomainTargetDiscoveryAsync() { List targetComputers; @@ -229,7 +230,7 @@ private void DomainTargetDiscovery() } // call ShareDisco which should handle the rest. - ShareDiscovery(targetComputers.ToArray()); + await ShareDiscoveryAsync(targetComputers.ToArray()); // ShareDiscovery(targetComputers.ToArray(), dfsShares); } @@ -285,12 +286,12 @@ public void PrepDomainUserRules() } } - private void ShareDiscovery(string[] computerTargets) + private async Task ShareDiscoveryAsync(string[] computerTargets) { Mq.Info("Starting to look for readable shares..."); foreach (string computer in computerTargets) { - if (CheckExclusions(computer)) + if (await CheckExclusions(computer)) { // skip any that are in the exclusion list continue; @@ -302,7 +303,7 @@ private void ShareDiscovery(string[] computerTargets) try { Mq.Trace("Performing reverse lookup for " + computer); - IPHostEntry result = Dns.GetHostEntry(computer); + IPHostEntry result = await Dns.GetHostEntryAsync(computer); computerName = result.HostName; Mq.Trace("Got DNSName " + computerName + " for " + computer); } @@ -343,7 +344,7 @@ public static bool isIP(string host) return IPAddress.TryParse(host, out ip); } - private bool CheckExclusions(string computer) + private async Task CheckExclusions(string computer) { // check if it's an IP already if (isIP(computer)) @@ -360,7 +361,7 @@ private bool CheckExclusions(string computer) try { // resolve it - IPHostEntry result = Dns.GetHostEntry(computer); + IPHostEntry result = await Dns.GetHostEntryAsync(computer); // handle multiple IPs in response foreach (IPAddress ipAddress in result.AddressList) { diff --git a/Snaffler/Config.cs b/Snaffler/Config.cs index 0a7e9a57..1d4e0914 100644 --- a/Snaffler/Config.cs +++ b/Snaffler/Config.cs @@ -8,38 +8,39 @@ using System.IO; using System.Linq; using System.Text; -using System.Reflection; -using System.Security; -using System.Collections.Generic; -using System.Net; +using System.Reflection; +using System.Security; +using System.Collections.Generic; +using System.Net; +using System.Threading.Tasks; namespace Snaffler { public static class Config { - public static Options Parse(string[] args) - { - BlockingMq Mq = BlockingMq.GetMq(); - Options options; - - // parse the args - try - { - options = ParseImpl(args); - if (options == null) - { - return null; - } - } - catch - { - Mq.Error("Something went wrong parsing args."); - throw; - } - - Mq.Info("Parsed args successfully."); - return options; - } + public static async Task ParseAsync(string[] args) + { + BlockingMq Mq = BlockingMq.GetMq(); + Options options; + + // parse the args + try + { + options = await ParseImplAsync(args); + if (options == null) + { + return null; + } + } + catch + { + Mq.Error("Something went wrong parsing args."); + throw; + } + + Mq.Info("Parsed args successfully."); + return options; + } public static string ReadResource(string name) @@ -62,11 +63,11 @@ public static bool isIP(string host) return IPAddress.TryParse(host, out ip); } - private static Options ParseImpl(string[] args) - { - BlockingMq Mq = BlockingMq.GetMq(); - Mq.Info("Parsing args..."); - Options parsedConfig = new Options(); + private static async Task ParseImplAsync(string[] args) + { + BlockingMq Mq = BlockingMq.GetMq(); + Mq.Info("Parsing args..."); + Options parsedConfig = new Options(); // define args ValueArgument configFileArg = new ValueArgument('z', "config", "Path to a .toml config file. Run with \'generate\' to puke a sample config file into the working directory."); @@ -198,26 +199,26 @@ private static Options ParseImpl(string[] args) { parsedConfig.DfsOnly = dfsArg.Value; } - if (compExclusionArg.Parsed) - { - List compExclusions = new List(); - string[] fileLines = File.ReadAllLines(compExclusionArg.Value); - foreach (string line in fileLines) - { - if (isIP(line)) - { - compExclusions.Add(line); - } - else - { - try - { - IPHostEntry result = Dns.GetHostEntry(line); - foreach (IPAddress ipAddress in result.AddressList) - { - compExclusions.Add(ipAddress.ToString()); - } - } + if (compExclusionArg.Parsed) + { + List compExclusions = new List(); + string[] fileLines = await File.ReadAllLinesAsync(compExclusionArg.Value); + foreach (string line in fileLines) + { + if (isIP(line)) + { + compExclusions.Add(line); + } + else + { + try + { + IPHostEntry result = await Dns.GetHostEntryAsync(line); + foreach (IPAddress ipAddress in result.AddressList) + { + compExclusions.Add(ipAddress.ToString()); + } + } catch (Exception ex) { Console.WriteLine(ex.Message); @@ -239,9 +240,10 @@ private static Options ParseImpl(string[] args) if (compTargetArg.Parsed) { List compTargets = new List(); - if (compTargetArg.Value.Contains(Path.DirectorySeparatorChar)) - { - compTargets.AddRange(File.ReadLines(compTargetArg.Value).Select(line => line.Trim())); + if (compTargetArg.Value.Contains(Path.DirectorySeparatorChar)) + { + string[] targetLines = await File.ReadAllLinesAsync(compTargetArg.Value); + compTargets.AddRange(targetLines.Select(line => line.Trim())); } else if (compTargetArg.Value.Contains(",")) { @@ -418,11 +420,11 @@ private static Options ParseImpl(string[] args) { string[] tomlfiles = Directory.GetFiles(parsedConfig.RuleDir, "*.toml", SearchOption.AllDirectories); StringBuilder sb = new StringBuilder(); - foreach (string tomlfile in tomlfiles) - { - string tomlstring = File.ReadAllText(tomlfile); - sb.AppendLine(tomlstring); - } + foreach (string tomlfile in tomlfiles) + { + string tomlstring = await File.ReadAllTextAsync(tomlfile); + sb.AppendLine(tomlstring); + } string bulktoml = sb.ToString(); // deserialise the toml to an actual ruleset RuleSet ruleSet = Toml.ReadString(bulktoml, settings); diff --git a/Snaffler/SnaffleRunner.cs b/Snaffler/SnaffleRunner.cs index 4d01882d..321ecff0 100644 --- a/Snaffler/SnaffleRunner.cs +++ b/Snaffler/SnaffleRunner.cs @@ -9,6 +9,7 @@ using System.Diagnostics; using System.IO; using System.Threading.Tasks; +using System.Threading.Tasks; using System.Text.RegularExpressions; using System.Threading; @@ -37,7 +38,7 @@ private string hostString() return _hostString; } - public void Run(string[] args) + public async Task RunAsync(string[] args) { // prime the hoststring lazy instantiator hostString(); @@ -52,7 +53,7 @@ public void Run(string[] args) try { // parse cli opts in - Options = Config.Parse(args); + Options = await Config.ParseAsync(args); if (Options == null) { @@ -199,18 +200,9 @@ public void Run(string[] args) controller = new SnaffCon(Options); - var tokenSource = new CancellationTokenSource(); - var token = tokenSource.Token; - Task thing = Task.Factory.StartNew(() => { controller.Execute(); }, token); - bool exit = false; - - while (exit == false) - { - if (HandleOutput() == true) - { - exit = true; - } - } + Task controllerTask = controller.ExecuteAsync(); + await HandleOutputAsync(); + await controllerTask; return; } catch (Exception e) @@ -223,7 +215,7 @@ public void Run(string[] args) private void DumpQueue() { BlockingMq Mq = BlockingMq.GetMq(); - while (Mq.Q.TryTake(out SnafflerMessage message)) + while (Mq.Reader.TryRead(out SnafflerMessage message)) { // emergency dump of queue contents to console Console.WriteLine(message.Message); @@ -234,10 +226,10 @@ private void DumpQueue() } } - private bool HandleOutput() + private async Task HandleOutputAsync() { BlockingMq Mq = BlockingMq.GetMq(); - foreach (SnafflerMessage message in Mq.Q.GetConsumingEnumerable()) + await foreach (SnafflerMessage message in Mq.Reader.ReadAllAsync()) { if (Options.LogType == LogType.Plain) { @@ -245,10 +237,10 @@ private bool HandleOutput() } else if (Options.LogType == LogType.JSON) { - ProcessMessageJSON(message); + await ProcessMessageJSONAsync(message); } - // catch terminating messages and bail out of the master 'while' loop + // catch terminating messages and bail out if ((message.Type == SnafflerMessageType.Fatal) || (message.Type == SnafflerMessageType.Finish)) { return true; @@ -304,7 +296,7 @@ private void ProcessMessage(SnafflerMessage message) } } - private void ProcessMessageJSON(SnafflerMessage message) + private async Task ProcessMessageJSONAsync(SnafflerMessage message) { // standardized time formatting, UTC string datetime = String.Format("{1}{0}{2:u}{0}", Options.Separator, hostString(), message.DateTime.ToUniversalTime()); @@ -355,10 +347,10 @@ private void ProcessMessageJSON(SnafflerMessage message) Console.WriteLine("Press any key to exit."); Console.ReadKey(); } - if (Options.LogType == LogType.JSON) + if (Options.LogType == LogType.JSON) { Logger.Info("Normalising output, please wait..."); - FixJSONOutput(); + await FixJSONOutputAsync(); } break; } @@ -553,29 +545,27 @@ public void PrintBanner() } //This is probably slow but it is a quick and easy fix for now. - private void FixJSONOutput() + private async Task FixJSONOutputAsync() { //Rename the log file temporarily File.Move(Options.LogFilePath, Options.LogFilePath + ".tmp"); //Prepare the normalised file - StreamWriter file = new StreamWriter(Options.LogFilePath); + using StreamWriter file = new StreamWriter(Options.LogFilePath); //Read in the original log file to an array - string[] lines = System.IO.File.ReadAllLines(Options.LogFilePath + ".tmp"); + string[] lines = await System.IO.File.ReadAllLinesAsync(Options.LogFilePath + ".tmp"); //Write the surrounding template that we need. - file.Write("{\"entries\": [\n"); + await file.WriteAsync("{\"entries\": [\n"); //Write all the lines into the new file but add a comma after all but the last so it becomes valid JSON. for (int ii = 0; ii < lines.Length -1; ii++) { - file.WriteLine(lines[ii] + ","); + await file.WriteLineAsync(lines[ii] + ","); } //Add the last line but without a comma - file.WriteLine(lines[lines.Length - 1]); + await file.WriteLineAsync(lines[lines.Length - 1]); //Close out the file's contents with the last of the JSON to make it valid. - file.Write("]\n}"); + await file.WriteAsync("]\n}"); //Flush the output - file.Flush(); - //Close the file - file.Close(); + await file.FlushAsync(); //Delete the temporary file. File.Delete(Options.LogFilePath + ".tmp"); } diff --git a/Snaffler/Snaffler.cs b/Snaffler/Snaffler.cs index dff04469..6db253e4 100644 --- a/Snaffler/Snaffler.cs +++ b/Snaffler/Snaffler.cs @@ -1,13 +1,14 @@ using System; +using System.Threading.Tasks; namespace Snaffler { public static class Snaffler { - public static void Main(string[] args) + public static async Task Main(string[] args) { SnaffleRunner runner = new SnaffleRunner(); - runner.Run(args); + await runner.RunAsync(args); Console.WriteLine("I snaffled 'til the snafflin was done."); } } From fae7eb4fc15ca8984dc867c310dc0f002dc9391d Mon Sep 17 00:00:00 2001 From: coj337 Date: Sat, 7 Jun 2025 12:59:55 +1000 Subject: [PATCH 04/10] Fix async blocking and add support for completing without waiting 5 minutes --- SnaffCore/SnaffCon.cs | 16 +++++ Snaffler/Config.cs | 124 +++++++++++++++++++------------------- Snaffler/SnaffleRunner.cs | 2 +- 3 files changed, 79 insertions(+), 63 deletions(-) diff --git a/SnaffCore/SnaffCon.cs b/SnaffCore/SnaffCon.cs index 8886344b..31a44d64 100644 --- a/SnaffCore/SnaffCon.cs +++ b/SnaffCore/SnaffCon.cs @@ -150,6 +150,8 @@ public async Task ExecuteAsync() Mq.Error("OctoParrot says: AWK! I SHOULDN'T BE!"); } + _ = StartCompletionWatcherAsync(); + waitHandle.WaitOne(); StatusUpdate(); @@ -499,6 +501,20 @@ private void StatusUpdate() //} } + private async Task StartCompletionWatcherAsync() + { + while (true) + { + if (FileTaskScheduler.Done() && ShareTaskScheduler.Done() && TreeTaskScheduler.Done()) + { + waitHandle.Set(); + break; + } + + await Task.Delay(1000); + } + } + private static String BytesToString(long byteCount) { string[] suf = { "B", "kB", "MB", "GB", "TB", "PB", "EB" }; //Longs run out around EB diff --git a/Snaffler/Config.cs b/Snaffler/Config.cs index 1d4e0914..1bf8d245 100644 --- a/Snaffler/Config.cs +++ b/Snaffler/Config.cs @@ -8,39 +8,39 @@ using System.IO; using System.Linq; using System.Text; -using System.Reflection; -using System.Security; -using System.Collections.Generic; -using System.Net; -using System.Threading.Tasks; +using System.Reflection; +using System.Security; +using System.Collections.Generic; +using System.Net; +using System.Threading.Tasks; namespace Snaffler { public static class Config { - public static async Task ParseAsync(string[] args) - { - BlockingMq Mq = BlockingMq.GetMq(); - Options options; - - // parse the args - try - { - options = await ParseImplAsync(args); - if (options == null) - { - return null; - } - } - catch - { - Mq.Error("Something went wrong parsing args."); - throw; - } - - Mq.Info("Parsed args successfully."); - return options; - } + public static async Task ParseAsync(string[] args) + { + BlockingMq Mq = BlockingMq.GetMq(); + Options options; + + // parse the args + try + { + options = await ParseImplAsync(args); + if (options == null) + { + return null; + } + } + catch + { + Mq.Error("Something went wrong parsing args."); + throw; + } + + Mq.Info("Parsed args successfully."); + return options; + } public static string ReadResource(string name) @@ -63,11 +63,11 @@ public static bool isIP(string host) return IPAddress.TryParse(host, out ip); } - private static async Task ParseImplAsync(string[] args) - { - BlockingMq Mq = BlockingMq.GetMq(); - Mq.Info("Parsing args..."); - Options parsedConfig = new Options(); + private static async Task ParseImplAsync(string[] args) + { + BlockingMq Mq = BlockingMq.GetMq(); + Mq.Info("Parsing args..."); + Options parsedConfig = new Options(); // define args ValueArgument configFileArg = new ValueArgument('z', "config", "Path to a .toml config file. Run with \'generate\' to puke a sample config file into the working directory."); @@ -199,26 +199,26 @@ private static async Task ParseImplAsync(string[] args) { parsedConfig.DfsOnly = dfsArg.Value; } - if (compExclusionArg.Parsed) - { - List compExclusions = new List(); - string[] fileLines = await File.ReadAllLinesAsync(compExclusionArg.Value); - foreach (string line in fileLines) - { - if (isIP(line)) - { - compExclusions.Add(line); - } - else - { - try - { - IPHostEntry result = await Dns.GetHostEntryAsync(line); - foreach (IPAddress ipAddress in result.AddressList) - { - compExclusions.Add(ipAddress.ToString()); - } - } + if (compExclusionArg.Parsed) + { + List compExclusions = new List(); + string[] fileLines = await File.ReadAllLinesAsync(compExclusionArg.Value); + foreach (string line in fileLines) + { + if (isIP(line)) + { + compExclusions.Add(line); + } + else + { + try + { + IPHostEntry result = await Dns.GetHostEntryAsync(line); + foreach (IPAddress ipAddress in result.AddressList) + { + compExclusions.Add(ipAddress.ToString()); + } + } catch (Exception ex) { Console.WriteLine(ex.Message); @@ -240,10 +240,10 @@ private static async Task ParseImplAsync(string[] args) if (compTargetArg.Parsed) { List compTargets = new List(); - if (compTargetArg.Value.Contains(Path.DirectorySeparatorChar)) - { - string[] targetLines = await File.ReadAllLinesAsync(compTargetArg.Value); - compTargets.AddRange(targetLines.Select(line => line.Trim())); + if (compTargetArg.Value.Contains(Path.DirectorySeparatorChar)) + { + string[] targetLines = await File.ReadAllLinesAsync(compTargetArg.Value); + compTargets.AddRange(targetLines.Select(line => line.Trim())); } else if (compTargetArg.Value.Contains(",")) { @@ -420,11 +420,11 @@ private static async Task ParseImplAsync(string[] args) { string[] tomlfiles = Directory.GetFiles(parsedConfig.RuleDir, "*.toml", SearchOption.AllDirectories); StringBuilder sb = new StringBuilder(); - foreach (string tomlfile in tomlfiles) - { - string tomlstring = await File.ReadAllTextAsync(tomlfile); - sb.AppendLine(tomlstring); - } + foreach (string tomlfile in tomlfiles) + { + string tomlstring = await File.ReadAllTextAsync(tomlfile); + sb.AppendLine(tomlstring); + } string bulktoml = sb.ToString(); // deserialise the toml to an actual ruleset RuleSet ruleSet = Toml.ReadString(bulktoml, settings); diff --git a/Snaffler/SnaffleRunner.cs b/Snaffler/SnaffleRunner.cs index 321ecff0..0917aea1 100644 --- a/Snaffler/SnaffleRunner.cs +++ b/Snaffler/SnaffleRunner.cs @@ -200,7 +200,7 @@ public async Task RunAsync(string[] args) controller = new SnaffCon(Options); - Task controllerTask = controller.ExecuteAsync(); + Task controllerTask = Task.Run(() => controller.ExecuteAsync()); await HandleOutputAsync(); await controllerTask; return; From 79cf5ce697745debcd16559a534cf484b884d87c Mon Sep 17 00:00:00 2001 From: coj337 Date: Sat, 7 Jun 2025 14:45:14 +1000 Subject: [PATCH 05/10] Fix hang on completion watcher --- SnaffCore/SnaffCon.cs | 2 +- Snaffler/Properties/launchSettings.json | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 Snaffler/Properties/launchSettings.json diff --git a/SnaffCore/SnaffCon.cs b/SnaffCore/SnaffCon.cs index 31a44d64..1af2c1bd 100644 --- a/SnaffCore/SnaffCon.cs +++ b/SnaffCore/SnaffCon.cs @@ -150,7 +150,7 @@ public async Task ExecuteAsync() Mq.Error("OctoParrot says: AWK! I SHOULDN'T BE!"); } - _ = StartCompletionWatcherAsync(); + _ = Task.Run(StartCompletionWatcherAsync); waitHandle.WaitOne(); diff --git a/Snaffler/Properties/launchSettings.json b/Snaffler/Properties/launchSettings.json new file mode 100644 index 00000000..12bee120 --- /dev/null +++ b/Snaffler/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "Snaffler": { + "commandName": "Project", + "commandLineArgs": "-s -i \"D:\\\\Backup - 2020-05-31\"" + } + } +} \ No newline at end of file From 3a624f4ba0ee92d118c66d7cd52b91a4017847be Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 9 Jun 2025 00:55:29 +0900 Subject: [PATCH 06/10] Handle redirected console for banner --- README.md | 11 + SnaffCore/Classifiers/ClassifierRule.cs | 16 +- SnaffCore/Config/ClassifierOptions.cs | 325 ++++----- SnaffCore/Config/Options.cs | 57 +- SnaffCore/SnaffCore.csproj | 4 +- SnaffCore/UltraSnaffCore.csproj | 4 +- Snaffler/Config.cs | 898 ++++++++++++------------ Snaffler/FodyWeavers.xml | 3 - Snaffler/FodyWeavers.xsd | 111 --- Snaffler/Logger.cs | 216 ++++++ Snaffler/RuleSet.cs | 7 +- Snaffler/SnaffleRunner.cs | 193 ++--- Snaffler/Snaffler.csproj | 5 +- 13 files changed, 962 insertions(+), 888 deletions(-) delete mode 100644 Snaffler/FodyWeavers.xml delete mode 100644 Snaffler/FodyWeavers.xsd create mode 100644 Snaffler/Logger.cs diff --git a/README.md b/README.md index 490b4c0e..ed3d45e6 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,17 @@ The solution was UltraSnaffler, which is just a second `.sln` file that enables WARNING: Snaffler's default rules don't include any that will look inside Office docs or PDFs, because we found it really difficult to write any that weren't going to just take *years* to finish a run in a typical corporate environment. *Be warned, looking inside these docs is a lot slower than looking inside good old fashioned text files, and a typical environment will have an absolute mountain of low-value Office docs and PDFs.* +## Building Native AOT executables + +Snaffler and its library projects now support publishing as native executables. +To create a self‑contained binary, run a command like: + +```bash +dotnet publish Snaffler/Snaffler.csproj -c Release -r win-x64 --self-contained +``` + +Replace the runtime identifier with the target OS and architecture of your choice. + ## How does the config file thing work? This is actually really neat IMO. diff --git a/SnaffCore/Classifiers/ClassifierRule.cs b/SnaffCore/Classifiers/ClassifierRule.cs index 4fb8837e..2830cbb3 100644 --- a/SnaffCore/Classifiers/ClassifierRule.cs +++ b/SnaffCore/Classifiers/ClassifierRule.cs @@ -1,30 +1,44 @@  using System.Collections.Generic; using System.Text.RegularExpressions; +using CsToml; namespace SnaffCore.Classifiers { - public class ClassifierRule + [TomlSerializedObject] + public partial class ClassifierRule { // define in what phase this rule is applied + [TomlValueOnSerialized] public EnumerationScope EnumerationScope { get; set; } = EnumerationScope.FileEnumeration; // define a way to chain rules together + [TomlValueOnSerialized] public string RuleName { get; set; } = "Default"; + [TomlValueOnSerialized] public MatchAction MatchAction { get; set; } = MatchAction.Snaffle; + [TomlValueOnSerialized] public List RelayTargets { get; set; } = null; + [TomlValueOnSerialized] public string Description { get; set; } = "A description of what a rule does."; // define the behaviour of this rule + [TomlValueOnSerialized] public MatchLoc MatchLocation { get; set; } = MatchLoc.FileName; + [TomlValueOnSerialized] public MatchListType WordListType { get; set; } = MatchListType.Contains; + [TomlValueOnSerialized] public int MatchLength { get; set; } = 0; + [TomlValueOnSerialized] public string MatchMD5 { get; set; } + [TomlValueOnSerialized] public List WordList { get; set; } = new List(); + [TomlValueOnSerialized] public List Regexes { get; set; } // define the severity of any matches + [TomlValueOnSerialized] public Triage Triage { get; set; } = Triage.Green; } diff --git a/SnaffCore/Config/ClassifierOptions.cs b/SnaffCore/Config/ClassifierOptions.cs index 42346fd4..843cd3c6 100644 --- a/SnaffCore/Config/ClassifierOptions.cs +++ b/SnaffCore/Config/ClassifierOptions.cs @@ -1,162 +1,165 @@ -using SnaffCore.Classifiers; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; - -namespace SnaffCore.Config -{ - public partial class Options - { - // Classifiers - public List ClassifierRules { get; set; } = new List(); - [Nett.TomlIgnore] - public List ShareClassifiers { get; set; } = new List(); - [Nett.TomlIgnore] - public List DirClassifiers { get; set; } = new List(); - [Nett.TomlIgnore] - public List FileClassifiers { get; set; } = new List(); - [Nett.TomlIgnore] - public List ContentsClassifiers { get; set; } = new List(); - [Nett.TomlIgnore] - public List PostMatchClassifiers { get; set; } = new List(); - - public void PrepareClassifiers() - { - // Where rules are using regexen, we precompile them here. - // We're gonna use them a lot so efficiency matters. - foreach (ClassifierRule classifierRule in ClassifierRules) - { - classifierRule.Regexes = new List(); - switch (classifierRule.WordListType) - { - case MatchListType.Regex: - foreach (string pattern in classifierRule.WordList) - { - classifierRule.Regexes.Add(new Regex(pattern, - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); - } - - break; - case MatchListType.Contains: - classifierRule.Regexes = new List(); - foreach (string pattern in classifierRule.WordList) - { - classifierRule.Regexes.Add(new Regex(pattern, - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); - } - - break; - case MatchListType.EndsWith: - foreach (string pattern in classifierRule.WordList) - { - classifierRule.Regexes.Add(new Regex(pattern + "$", - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); - } - - break; - case MatchListType.StartsWith: - foreach (string pattern in classifierRule.WordList) - { - classifierRule.Regexes.Add(new Regex("^" + pattern, - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); - } - - break; - case MatchListType.Exact: - foreach (string pattern in classifierRule.WordList) - { - classifierRule.Regexes.Add(new Regex("^" + pattern + "$", - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); - } - - break; - - } - } - - // figure out which rules match our interest level flag - ClassifierRules = (from classifier in ClassifierRules - where IsInterest(classifier) - select classifier).ToList(); - // sort everything into enumeration scopes - ShareClassifiers = (from classifier in ClassifierRules - where classifier.EnumerationScope == EnumerationScope.ShareEnumeration - select classifier).ToList(); - // and sort them by Match type - ShareClassifiers.Sort((x, y) => x.MatchAction.CompareTo(y.MatchAction)); - DirClassifiers = (from classifier in ClassifierRules - where classifier.EnumerationScope == EnumerationScope.DirectoryEnumeration - select classifier).ToList(); - DirClassifiers.Sort((x, y) => x.MatchAction.CompareTo(y.MatchAction)); - FileClassifiers = (from classifier in ClassifierRules - where classifier.EnumerationScope == EnumerationScope.FileEnumeration - select classifier).ToList(); - FileClassifiers.Sort((x, y) => x.MatchAction.CompareTo(y.MatchAction)); - ContentsClassifiers = (from classifier in ClassifierRules - where classifier.EnumerationScope == EnumerationScope.ContentsEnumeration - select classifier).ToList(); - ContentsClassifiers.Sort((x, y) => x.MatchAction.CompareTo(y.MatchAction)); - PostMatchClassifiers = (from classifier in ClassifierRules - where classifier.EnumerationScope == EnumerationScope.PostMatch - select classifier).ToList(); - } - - private bool IsInterest(ClassifierRule classifier) - { - /* - * Keep all discard & archive parsing rules. - * Else, if rule (or child rule, recursive) interest level is lower than provided (0 default), then discard - */ - try - { - - if (classifier.RelayTargets != null) - { - int max = 0; - foreach (string relayTarget in classifier.RelayTargets) - { - try - { - ClassifierRule relayRule = ClassifierRules.First(thing => thing.RuleName == relayTarget); - - if ( - (relayRule.Triage == Triage.Black && InterestLevel > 3) || - (relayRule.Triage == Triage.Red && InterestLevel > 2) || - (relayRule.Triage == Triage.Yellow && InterestLevel > 1) || - (relayRule.Triage == Triage.Green && InterestLevel > 0)) - { - return true; - } - } - catch (Exception e) - { - throw new Exception("You have a misconfigured rule trying to relay to " + relayTarget + " and no such rule exists by that name."); - } - } - } - - - bool actualThing = !( - ( - classifier.MatchAction == MatchAction.Snaffle || - classifier.MatchAction == MatchAction.CheckForKeys - ) && - ( - (classifier.Triage == Triage.Black && InterestLevel > 3) || - (classifier.Triage == Triage.Red && InterestLevel > 2) || - (classifier.Triage == Triage.Yellow && InterestLevel > 1) || - (classifier.Triage == Triage.Green && InterestLevel > 0) - ) - ); - return actualThing; - } - catch (Exception e) - { - Console.WriteLine(classifier.RuleName); - Console.WriteLine(e.ToString()); - } - return true; - } - } +using SnaffCore.Classifiers; +using System; +using System.Collections.Generic; +using System.Linq; +using CsToml; +using System.Runtime.Serialization; +using System.Text.RegularExpressions; + +namespace SnaffCore.Config +{ + public partial class Options + { + // Classifiers + [TomlValueOnSerialized] + public List ClassifierRules { get; set; } = new List(); + [IgnoreDataMember] + public List ShareClassifiers { get; set; } = new List(); + [IgnoreDataMember] + public List DirClassifiers { get; set; } = new List(); + [IgnoreDataMember] + public List FileClassifiers { get; set; } = new List(); + [IgnoreDataMember] + public List ContentsClassifiers { get; set; } = new List(); + [IgnoreDataMember] + public List PostMatchClassifiers { get; set; } = new List(); + + public void PrepareClassifiers() + { + // Where rules are using regexen, we precompile them here. + // We're gonna use them a lot so efficiency matters. + foreach (ClassifierRule classifierRule in ClassifierRules) + { + classifierRule.Regexes = new List(); + switch (classifierRule.WordListType) + { + case MatchListType.Regex: + foreach (string pattern in classifierRule.WordList) + { + classifierRule.Regexes.Add(new Regex(pattern, + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); + } + + break; + case MatchListType.Contains: + classifierRule.Regexes = new List(); + foreach (string pattern in classifierRule.WordList) + { + classifierRule.Regexes.Add(new Regex(pattern, + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); + } + + break; + case MatchListType.EndsWith: + foreach (string pattern in classifierRule.WordList) + { + classifierRule.Regexes.Add(new Regex(pattern + "$", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); + } + + break; + case MatchListType.StartsWith: + foreach (string pattern in classifierRule.WordList) + { + classifierRule.Regexes.Add(new Regex("^" + pattern, + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); + } + + break; + case MatchListType.Exact: + foreach (string pattern in classifierRule.WordList) + { + classifierRule.Regexes.Add(new Regex("^" + pattern + "$", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); + } + + break; + + } + } + + // figure out which rules match our interest level flag + ClassifierRules = (from classifier in ClassifierRules + where IsInterest(classifier) + select classifier).ToList(); + // sort everything into enumeration scopes + ShareClassifiers = (from classifier in ClassifierRules + where classifier.EnumerationScope == EnumerationScope.ShareEnumeration + select classifier).ToList(); + // and sort them by Match type + ShareClassifiers.Sort((x, y) => x.MatchAction.CompareTo(y.MatchAction)); + DirClassifiers = (from classifier in ClassifierRules + where classifier.EnumerationScope == EnumerationScope.DirectoryEnumeration + select classifier).ToList(); + DirClassifiers.Sort((x, y) => x.MatchAction.CompareTo(y.MatchAction)); + FileClassifiers = (from classifier in ClassifierRules + where classifier.EnumerationScope == EnumerationScope.FileEnumeration + select classifier).ToList(); + FileClassifiers.Sort((x, y) => x.MatchAction.CompareTo(y.MatchAction)); + ContentsClassifiers = (from classifier in ClassifierRules + where classifier.EnumerationScope == EnumerationScope.ContentsEnumeration + select classifier).ToList(); + ContentsClassifiers.Sort((x, y) => x.MatchAction.CompareTo(y.MatchAction)); + PostMatchClassifiers = (from classifier in ClassifierRules + where classifier.EnumerationScope == EnumerationScope.PostMatch + select classifier).ToList(); + } + + private bool IsInterest(ClassifierRule classifier) + { + /* + * Keep all discard & archive parsing rules. + * Else, if rule (or child rule, recursive) interest level is lower than provided (0 default), then discard + */ + try + { + + if (classifier.RelayTargets != null) + { + int max = 0; + foreach (string relayTarget in classifier.RelayTargets) + { + try + { + ClassifierRule relayRule = ClassifierRules.First(thing => thing.RuleName == relayTarget); + + if ( + (relayRule.Triage == Triage.Black && InterestLevel > 3) || + (relayRule.Triage == Triage.Red && InterestLevel > 2) || + (relayRule.Triage == Triage.Yellow && InterestLevel > 1) || + (relayRule.Triage == Triage.Green && InterestLevel > 0)) + { + return true; + } + } + catch (Exception e) + { + throw new Exception("You have a misconfigured rule trying to relay to " + relayTarget + " and no such rule exists by that name."); + } + } + } + + + bool actualThing = !( + ( + classifier.MatchAction == MatchAction.Snaffle || + classifier.MatchAction == MatchAction.CheckForKeys + ) && + ( + (classifier.Triage == Triage.Black && InterestLevel > 3) || + (classifier.Triage == Triage.Red && InterestLevel > 2) || + (classifier.Triage == Triage.Yellow && InterestLevel > 1) || + (classifier.Triage == Triage.Green && InterestLevel > 0) + ) + ); + return actualThing; + } + catch (Exception e) + { + Console.WriteLine(classifier.RuleName); + Console.WriteLine(e.ToString()); + } + return true; + } + } } \ No newline at end of file diff --git a/SnaffCore/Config/Options.cs b/SnaffCore/Config/Options.cs index 78c66282..8967c689 100644 --- a/SnaffCore/Config/Options.cs +++ b/SnaffCore/Config/Options.cs @@ -1,6 +1,8 @@ -using System.Collections.Generic; -using System.Security.Principal; -using SnaffCore.ActiveDirectory; +using System.Collections.Generic; +using System.Security.Principal; +using System.Runtime.Serialization; +using SnaffCore.ActiveDirectory; +using CsToml; namespace SnaffCore.Config { @@ -10,57 +12,94 @@ public enum LogType JSON = 1 } + [TomlSerializedObject] public partial class Options { public static Options MyOptions { get; set; } // Manual Targeting Options + [TomlValueOnSerialized] public List PathTargets { get; set; } = new List(); + [TomlValueOnSerialized] public string[] ComputerTargets { get; set; } + [TomlValueOnSerialized] public string ComputerTargetsLdapFilter { get; set; } = "(objectClass=computer)"; + [TomlValueOnSerialized] public string ComputerExclusionFile { get; set; } + [TomlValueOnSerialized] public List ComputerExclusions { get; set; } = new List(); + [TomlValueOnSerialized] public bool ScanSysvol { get; set; } = true; + [TomlValueOnSerialized] public bool ScanNetlogon { get; set; } = true; + [TomlValueOnSerialized] public bool ScanFoundShares { get; set; } = true; + [TomlValueOnSerialized] public int InterestLevel { get; set; } = 0; + [TomlValueOnSerialized] public bool DfsOnly { get; set; } = false; + [TomlValueOnSerialized] public bool DfsShareDiscovery { get; set; } = false; - [Nett.TomlIgnore] - public Dictionary DfsSharesDict { get; set; } = new Dictionary(); + [IgnoreDataMember] + public Dictionary DfsSharesDict { get; set; } = new Dictionary(); + [TomlValueOnSerialized] public List DfsNamespacePaths { get; set; } = new List(); + [TomlValueOnSerialized] public string CurrentUser { get; set; } = WindowsIdentity.GetCurrent().Name; + [TomlValueOnSerialized] public string RuleDir { get; set; } + [TomlValueOnSerialized] public int TimeOut { get; set; } = 5; // Concurrency Options + [TomlValueOnSerialized] public int MaxThreads { get; set; } = 60; + [TomlValueOnSerialized] public int ShareThreads { get; set; } + [TomlValueOnSerialized] public int TreeThreads { get; set; } + [TomlValueOnSerialized] public int FileThreads { get; set; } + [TomlValueOnSerialized] public int MaxFileQueue { get; set; } = 200000; + [TomlValueOnSerialized] public int MaxTreeQueue { get; set; } = 0; + [TomlValueOnSerialized] public int MaxShareQueue { get; set; } = 0; // Logging Options + [TomlValueOnSerialized] public bool LogToFile { get; set; } = false; + [TomlValueOnSerialized] public string LogFilePath { get; set; } + [TomlValueOnSerialized] public LogType LogType { get; set; } + [TomlValueOnSerialized] public bool LogTSV { get; set; } = false; + [TomlValueOnSerialized] public char Separator { get; set; } = ' '; + [TomlValueOnSerialized] public bool LogToConsole { get; set; } = true; + [TomlValueOnSerialized] public string LogLevelString { get; set; } = "info"; // ShareFinder Options + [TomlValueOnSerialized] public bool ShareFinderEnabled { get; set; } = true; + [TomlValueOnSerialized] public string TargetDomain { get; set; } + [TomlValueOnSerialized] public string TargetDc { get; set; } + [TomlValueOnSerialized] public bool LogDeniedShares { get; set; } = false; // FileScanner Options + [TomlValueOnSerialized] public bool DomainUserRules { get; set; } = false; + [TomlValueOnSerialized] public int DomainUserMinLen { get; set; } = 6; + [TomlValueOnSerialized] public DomainUserNamesFormat[] DomainUserNameFormats { get; set; } = new DomainUserNamesFormat[] { DomainUserNamesFormat.sAMAccountName }; // passwords to try on certs that require one @@ -90,6 +129,7 @@ public partial class Options public List DomainUsersToMatch = new List(); // These options can be set in toml. They need the get/set accessor + [TomlValueOnSerialized] public List DomainUserMatchStrings { get; set; } = new List() { "sql", @@ -109,22 +149,29 @@ public partial class Options "configmgr" }; + [TomlValueOnSerialized] public List DomainUserStrictStrings { get; set; } + [TomlValueOnSerialized] public List DomainUsersWordlistRules { get; set; } = new List() { "KeepConfigRegexRed" }; // this sets the maximum size of file to look inside. + [TomlValueOnSerialized] public long MaxSizeToGrep { get; set; } = 1000000; // these enable or disable automated downloading of files that match the criteria + [TomlValueOnSerialized] public bool Snaffle { get; set; } = false; + [TomlValueOnSerialized] public long MaxSizeToSnaffle { get; set; } = 10000000; + [TomlValueOnSerialized] public string SnafflePath { get; set; } // Content processing options + [TomlValueOnSerialized] public int MatchContextBytes { get; set; } = 200; public Options() diff --git a/SnaffCore/SnaffCore.csproj b/SnaffCore/SnaffCore.csproj index 2c86f732..1ff530a6 100644 --- a/SnaffCore/SnaffCore.csproj +++ b/SnaffCore/SnaffCore.csproj @@ -2,9 +2,11 @@ net9.0 false + true - + + diff --git a/SnaffCore/UltraSnaffCore.csproj b/SnaffCore/UltraSnaffCore.csproj index fa38f498..1fac9e66 100644 --- a/SnaffCore/UltraSnaffCore.csproj +++ b/SnaffCore/UltraSnaffCore.csproj @@ -3,9 +3,11 @@ net9.0 false ULTRASNAFFLER + true - + + diff --git a/Snaffler/Config.cs b/Snaffler/Config.cs index 1bf8d245..39c38bcf 100644 --- a/Snaffler/Config.cs +++ b/Snaffler/Config.cs @@ -1,451 +1,447 @@ -using CommandLineParser.Arguments; -using Nett; -using NLog; -using SnaffCore.Concurrency; -using SnaffCore.Config; -using System; -using System.Resources; -using System.IO; -using System.Linq; -using System.Text; -using System.Reflection; -using System.Security; -using System.Collections.Generic; -using System.Net; -using System.Threading.Tasks; - -namespace Snaffler -{ - public static class Config - { - public static async Task ParseAsync(string[] args) - { - BlockingMq Mq = BlockingMq.GetMq(); - Options options; - - // parse the args - try - { - options = await ParseImplAsync(args); - if (options == null) - { - return null; - } - } - catch - { - Mq.Error("Something went wrong parsing args."); - throw; - } - - Mq.Info("Parsed args successfully."); - return options; - } - - - public static string ReadResource(string name) - { - // Determine path - var assembly = Assembly.GetExecutingAssembly(); - string resourcePath = name; - // Format: "{Namespace}.{Folder}.{filename}.{Extension}" - - using (Stream stream = assembly.GetManifestResourceStream(resourcePath)) - using (StreamReader reader = new StreamReader(stream)) - { - return reader.ReadToEnd(); - } - } - - public static bool isIP(string host) - { - IPAddress ip; - return IPAddress.TryParse(host, out ip); - } - - private static async Task ParseImplAsync(string[] args) - { - BlockingMq Mq = BlockingMq.GetMq(); - Mq.Info("Parsing args..."); - Options parsedConfig = new Options(); - - // define args - ValueArgument configFileArg = new ValueArgument('z', "config", "Path to a .toml config file. Run with \'generate\' to puke a sample config file into the working directory."); - ValueArgument outFileArg = new ValueArgument('o', "outfile", - "Path for output file. You probably want this if you're not using -s."); - ValueArgument verboseArg = new ValueArgument('v', "verbosity", - "Controls verbosity level, options are Trace (most verbose), Debug (less verbose), Info (less verbose still, default), and Data (results only). e.g '-v debug' "); - SwitchArgument helpArg = new SwitchArgument('h', "help", "Displays this help.", false); - SwitchArgument stdOutArg = new SwitchArgument('s', "stdout", - "Enables outputting results to stdout as soon as they're found. You probably want this if you're not using -o.", - false); - ValueArgument interestLevel = new ValueArgument('b', "interest", "Interest level to report (0-3)"); - ValueArgument snaffleArg = new ValueArgument('m', "snaffle", - "Enables and assigns an output dir for Snaffler to automatically snaffle a copy of any found files."); - ValueArgument snaffleSizeArg = new ValueArgument('l', "snafflesize", "Maximum size of file to snaffle, in bytes. Defaults to 10MB."); - //var fileHuntArg = new SwitchArgument('f', "filehuntoff", - // "Disables file discovery, will only perform computer and share discovery.", false); - ValueArgument dirTargetArg = new ValueArgument('i', "dirtarget", - "Disables computer and share discovery, requires a path to a directory in which to perform file discovery."); - ValueArgument domainArg = new ValueArgument('d', "domain", - "Domain to search for computers to search for shares on to search for files in. Easy."); - ValueArgument domainControllerArg = new ValueArgument('c', "domaincontroller", - "Domain controller to query for a list of domain computers."); - ValueArgument maxGrepSizeArg = new ValueArgument('r', "maxgrepsize", - "The maximum size file (in bytes) to search inside for interesting strings. Defaults to 500k."); - ValueArgument grepContextArg = new ValueArgument('j', "grepcontext", - "How many bytes of context either side of found strings in files to show, e.g. -j 200"); - SwitchArgument domainUserArg = new SwitchArgument('u', "domainusers", "Makes Snaffler grab a list of interesting-looking accounts from the domain and uses them in searches.", false); - ValueArgument maxThreadsArg = new ValueArgument('x', "maxthreads", "How many threads to be snaffling with. Any less than 4 and you're gonna have a bad time."); - SwitchArgument tsvArg = new SwitchArgument('y', "tsv", "Makes Snaffler output as tsv.", false); - SwitchArgument dfsArg = new SwitchArgument('f', "dfs", "Limits Snaffler to finding file shares via DFS, for \"OPSEC\" reasons.", false); - SwitchArgument findSharesOnlyArg = new SwitchArgument('a', "sharesonly", - "Stops after finding shares, doesn't walk their filesystems.", false); - ValueArgument compExclusionArg = new ValueArgument('k', "exclusions", "Path to a file containing a list of computers to exclude from scanning."); - ValueArgument compTargetArg = new ValueArgument('n', "comptarget", "List of computers in a file(e.g C:\targets.txt), a single Computer (or comma separated list) to target."); - ValueArgument ruleDirArg = new ValueArgument('p', "rulespath", "Path to a directory full of toml-formatted rules. Snaffler will load all of these in place of the default ruleset."); - ValueArgument logType = new ValueArgument('t', "logtype", "Type of log you would like to output. Currently supported options are plain and JSON. Defaults to plain."); - ValueArgument timeOutArg = new ValueArgument('e', "timeout", - "Interval between status updates (in minutes) also acts as a timeout for AD data to be gathered via LDAP. Turn this knob up if you aren't getting any computers from AD when you run Snaffler through a proxy or other slow link. Default = 5"); - // list of letters i haven't used yet: gnqw - - CommandLineParser.CommandLineParser parser = new CommandLineParser.CommandLineParser(); - parser.Arguments.Add(timeOutArg); - parser.Arguments.Add(configFileArg); - parser.Arguments.Add(outFileArg); - parser.Arguments.Add(helpArg); - parser.Arguments.Add(stdOutArg); - parser.Arguments.Add(snaffleArg); - parser.Arguments.Add(snaffleSizeArg); - parser.Arguments.Add(dirTargetArg); - parser.Arguments.Add(interestLevel); - parser.Arguments.Add(domainArg); - parser.Arguments.Add(verboseArg); - parser.Arguments.Add(domainControllerArg); - parser.Arguments.Add(maxGrepSizeArg); - parser.Arguments.Add(grepContextArg); - parser.Arguments.Add(domainUserArg); - parser.Arguments.Add(tsvArg); - parser.Arguments.Add(dfsArg); - parser.Arguments.Add(findSharesOnlyArg); - parser.Arguments.Add(maxThreadsArg); - parser.Arguments.Add(compTargetArg); - parser.Arguments.Add(ruleDirArg); - parser.Arguments.Add(logType); - parser.Arguments.Add(compExclusionArg); - - // extra check to handle builtin behaviour from cmd line arg parser - if ((args.Contains("--help") || args.Contains("/?") || args.Contains("help") || args.Contains("-h") || args.Length == 0)) - { - parser.ShowUsage(); - return null; - } - - TomlSettings settings = TomlSettings.Create(cfg => cfg -.ConfigureType(tc => - tc.WithConversionFor(conv => conv - .FromToml(s => (LogLevel)Enum.Parse(typeof(LogLevel), s.Value, ignoreCase: true)) - .ToToml(e => e.ToString())))); - - try - { - parser.ParseCommandLine(args); - - if (timeOutArg.Parsed && !String.IsNullOrWhiteSpace(timeOutArg.Value)) - { - int timeOutVal; - if (int.TryParse(timeOutArg.Value, out timeOutVal)) - { - Mq.Info("Set timeout/update interval to " + timeOutVal.ToString() + " minutes."); - parsedConfig.TimeOut = timeOutVal; - } - else - { - Mq.Error("Invalid timeout value passed, defaulting to 5 mins."); - } - } - - if (logType.Parsed && !String.IsNullOrWhiteSpace(logType.Value)) - { - //Set the default to plain - parsedConfig.LogType = LogType.Plain; - //if they set a different type then replace it with the new type. - if (logType.Value.ToLower() == "json") - { - parsedConfig.LogType = LogType.JSON; - } - else - { - Mq.Info("Invalid type argument passed (" + logType.Value + ") defaulting to plaintext"); - } - } - - if (ruleDirArg.Parsed && !String.IsNullOrWhiteSpace(ruleDirArg.Value)) - { - parsedConfig.RuleDir = ruleDirArg.Value; - } - - // get the args into our config - - // output args - if (outFileArg.Parsed && (!String.IsNullOrEmpty(outFileArg.Value))) - { - parsedConfig.LogToFile = true; - parsedConfig.LogFilePath = outFileArg.Value; - Mq.Degub("Logging to file at " + parsedConfig.LogFilePath); - } - - if (dfsArg.Parsed) - { - parsedConfig.DfsOnly = dfsArg.Value; - } - if (compExclusionArg.Parsed) - { - List compExclusions = new List(); - string[] fileLines = await File.ReadAllLinesAsync(compExclusionArg.Value); - foreach (string line in fileLines) - { - if (isIP(line)) - { - compExclusions.Add(line); - } - else - { - try - { - IPHostEntry result = await Dns.GetHostEntryAsync(line); - foreach (IPAddress ipAddress in result.AddressList) - { - compExclusions.Add(ipAddress.ToString()); - } - } - catch (Exception ex) - { - Console.WriteLine(ex.Message); - continue; - } - } - } - if (compExclusions.Count > 0) - { - parsedConfig.ComputerExclusions = compExclusions; - parsedConfig.ComputerExclusionFile = compExclusionArg.Value; - } - else - { - throw new Exception("Failed to get a valid list of excluded computers from the excluded computers list."); - } - } - - if (compTargetArg.Parsed) - { - List compTargets = new List(); - if (compTargetArg.Value.Contains(Path.DirectorySeparatorChar)) - { - string[] targetLines = await File.ReadAllLinesAsync(compTargetArg.Value); - compTargets.AddRange(targetLines.Select(line => line.Trim())); - } - else if (compTargetArg.Value.Contains(",")) - { - compTargets.AddRange(compTargetArg.Value.Split(',').Select(x => x.Trim())); - } - else - { - compTargets.Add(compTargetArg.Value.Trim()); - } - parsedConfig.ComputerTargets = compTargets.ToArray(); - } - - if (findSharesOnlyArg.Parsed) - { - parsedConfig.ScanFoundShares = false; - } - if (maxThreadsArg.Parsed) - { - parsedConfig.MaxThreads = maxThreadsArg.Value; - } - - parsedConfig.ShareThreads = parsedConfig.MaxThreads / 3; - parsedConfig.FileThreads = parsedConfig.MaxThreads / 3; - parsedConfig.TreeThreads = parsedConfig.MaxThreads / 3; - - if (tsvArg.Parsed) - { - parsedConfig.LogTSV = true; - if (parsedConfig.Separator == ' ') - { - parsedConfig.Separator = '\t'; - } - } - - // Set loglevel. - if (verboseArg.Parsed) - { - parsedConfig.LogLevelString = verboseArg.Value; - Mq.Degub("Requested verbosity level: " + parsedConfig.LogLevelString); - } - - // if enabled, display findings to the console - parsedConfig.LogToConsole = stdOutArg.Parsed; - Mq.Degub("Enabled logging to stdout."); - - // args that tell us about targeting - if ((domainArg.Parsed) && (!String.IsNullOrEmpty(domainArg.Value))) - { - parsedConfig.TargetDomain = domainArg.Value; - Mq.Degub("Target domain is " + domainArg.Value); - } - - if ((domainControllerArg.Parsed) && (!String.IsNullOrEmpty(domainControllerArg.Value))) - { - parsedConfig.TargetDc = domainControllerArg.Value; - Mq.Degub("Target DC is " + domainControllerArg.Value); - } - - if (domainUserArg.Parsed) - { - parsedConfig.DomainUserRules = true; - Mq.Degub("Enabled use of domain user accounts in rules."); - } - - if (dirTargetArg.Parsed) - { - parsedConfig.ShareFinderEnabled = false; - //Console.WriteLine(dirTargetArg.Value); - string pathTarget = dirTargetArg.Value; - if (dirTargetArg.Value.Length > 4) - { - pathTarget = dirTargetArg.Value.TrimEnd('\\'); - } - parsedConfig.PathTargets.Add(pathTarget); - //Console.WriteLine(parsedConfig.PathTargets[0]); - Mq.Degub("Disabled finding shares."); - Mq.Degub("Target path is " + dirTargetArg.Value); - } - - if (maxGrepSizeArg.Parsed) - { - parsedConfig.MaxSizeToGrep = maxGrepSizeArg.Value; - Mq.Degub("We won't bother looking inside files if they're bigger than " + parsedConfig.MaxSizeToGrep + - " bytes"); - } - - if (snaffleSizeArg.Parsed) - { - parsedConfig.MaxSizeToSnaffle = snaffleSizeArg.Value; - } - - if (interestLevel.Parsed) - { - parsedConfig.InterestLevel = interestLevel.Value; - Mq.Degub("Requested interest level: " + parsedConfig.InterestLevel); - } - - // how many bytes - if (grepContextArg.Parsed) - { - parsedConfig.MatchContextBytes = grepContextArg.Value; - Mq.Degub( - "We'll show you " + grepContextArg.Value + - " bytes of context around matches inside files."); - } - - // if enabled, grab a copy of files that we like. - if (snaffleArg.Parsed) - { - if (snaffleArg.Value.Length <= 0) - { - Mq.Error("-m or -mirror arg requires a path value."); - throw new ArgumentException("Invalid argument combination."); - } - - parsedConfig.Snaffle = true; - parsedConfig.SnafflePath = snaffleArg.Value.TrimEnd('\\'); - Mq.Degub("Mirroring matched files to path " + parsedConfig.SnafflePath); - } - - if (configFileArg.Parsed) - { - if (configFileArg.Value.Equals("generate")) - { - Toml.WriteFile(parsedConfig, ".\\default.toml", settings); - Console.WriteLine("Wrote config values to .\\default.toml"); - parsedConfig.LogToConsole = true; - Mq.Degub("Enabled logging to stdout."); - return null; - } - else - { - string configFile = configFileArg.Value; - parsedConfig = Toml.ReadFile(configFile, settings); - Mq.Info("Read config file from " + configFile); - } - } - - if (!parsedConfig.LogToConsole && !parsedConfig.LogToFile) - { - Mq.Error( - "\nYou didn't enable output to file or to the console so you won't see any results or debugs or anything. Your l0ss."); - throw new ArgumentException("Pointless argument combination."); - } - - if (parsedConfig.ClassifierRules.Count <= 0) - { - if (String.IsNullOrWhiteSpace(parsedConfig.RuleDir)) - { - // get all the embedded toml file resources - string[] resourceNames = Assembly.GetExecutingAssembly().GetManifestResourceNames(); - StringBuilder sb = new StringBuilder(); - - foreach (string resourceName in resourceNames) - { - if (!resourceName.EndsWith(".toml")) - { - // skip this one as it's just metadata - continue; - } - string ruleFile = ReadResource(resourceName); - sb.AppendLine(ruleFile); - } - - string bulktoml = sb.ToString(); - - // deserialise the toml to an actual ruleset - RuleSet ruleSet = Toml.ReadString(bulktoml, settings); - - // stick the rules in our config! - parsedConfig.ClassifierRules = ruleSet.ClassifierRules; - } - else - { - string[] tomlfiles = Directory.GetFiles(parsedConfig.RuleDir, "*.toml", SearchOption.AllDirectories); - StringBuilder sb = new StringBuilder(); - foreach (string tomlfile in tomlfiles) - { - string tomlstring = await File.ReadAllTextAsync(tomlfile); - sb.AppendLine(tomlstring); - } - string bulktoml = sb.ToString(); - // deserialise the toml to an actual ruleset - RuleSet ruleSet = Toml.ReadString(bulktoml, settings); - - // stick the rules in our config! - parsedConfig.ClassifierRules = ruleSet.ClassifierRules; - } - } - - parsedConfig.PrepareClassifiers(); - } - catch (Exception e) - { - Mq.Error(e.ToString()); - throw; - } - - return parsedConfig; - } - - - - } -} +using CommandLineParser.Arguments; +using CsToml; +using SnaffCore.Concurrency; +using SnaffCore.Config; +using System; +using System.Resources; +using System.IO; +using System.Linq; +using System.Text; +using System.Reflection; +using System.Security; +using System.Collections.Generic; +using System.Net; +using System.Threading.Tasks; + +namespace Snaffler +{ + public static class Config + { + + public static async Task ParseAsync(string[] args) + { + BlockingMq Mq = BlockingMq.GetMq(); + Options options; + + // parse the args + try + { + options = await ParseImplAsync(args); + if (options == null) + { + return null; + } + } + catch + { + Mq.Error("Something went wrong parsing args."); + throw; + } + + Mq.Info("Parsed args successfully."); + return options; + } + + + public static string ReadResource(string name) + { + // Determine path + var assembly = Assembly.GetExecutingAssembly(); + string resourcePath = name; + // Format: "{Namespace}.{Folder}.{filename}.{Extension}" + + using (Stream stream = assembly.GetManifestResourceStream(resourcePath)) + using (StreamReader reader = new StreamReader(stream)) + { + return reader.ReadToEnd(); + } + } + + public static bool isIP(string host) + { + IPAddress ip; + return IPAddress.TryParse(host, out ip); + } + + private static async Task ParseImplAsync(string[] args) + { + BlockingMq Mq = BlockingMq.GetMq(); + Mq.Info("Parsing args..."); + Options parsedConfig = new Options(); + + // define args + ValueArgument configFileArg = new ValueArgument('z', "config", "Path to a .toml config file. Run with \'generate\' to puke a sample config file into the working directory."); + ValueArgument outFileArg = new ValueArgument('o', "outfile", + "Path for output file. You probably want this if you're not using -s."); + ValueArgument verboseArg = new ValueArgument('v', "verbosity", + "Controls verbosity level, options are Trace (most verbose), Debug (less verbose), Info (less verbose still, default), and Data (results only). e.g '-v debug' "); + SwitchArgument helpArg = new SwitchArgument('h', "help", "Displays this help.", false); + SwitchArgument stdOutArg = new SwitchArgument('s', "stdout", + "Enables outputting results to stdout as soon as they're found. You probably want this if you're not using -o.", + false); + ValueArgument interestLevel = new ValueArgument('b', "interest", "Interest level to report (0-3)"); + ValueArgument snaffleArg = new ValueArgument('m', "snaffle", + "Enables and assigns an output dir for Snaffler to automatically snaffle a copy of any found files."); + ValueArgument snaffleSizeArg = new ValueArgument('l', "snafflesize", "Maximum size of file to snaffle, in bytes. Defaults to 10MB."); + //var fileHuntArg = new SwitchArgument('f', "filehuntoff", + // "Disables file discovery, will only perform computer and share discovery.", false); + ValueArgument dirTargetArg = new ValueArgument('i', "dirtarget", + "Disables computer and share discovery, requires a path to a directory in which to perform file discovery."); + ValueArgument domainArg = new ValueArgument('d', "domain", + "Domain to search for computers to search for shares on to search for files in. Easy."); + ValueArgument domainControllerArg = new ValueArgument('c', "domaincontroller", + "Domain controller to query for a list of domain computers."); + ValueArgument maxGrepSizeArg = new ValueArgument('r', "maxgrepsize", + "The maximum size file (in bytes) to search inside for interesting strings. Defaults to 500k."); + ValueArgument grepContextArg = new ValueArgument('j', "grepcontext", + "How many bytes of context either side of found strings in files to show, e.g. -j 200"); + SwitchArgument domainUserArg = new SwitchArgument('u', "domainusers", "Makes Snaffler grab a list of interesting-looking accounts from the domain and uses them in searches.", false); + ValueArgument maxThreadsArg = new ValueArgument('x', "maxthreads", "How many threads to be snaffling with. Any less than 4 and you're gonna have a bad time."); + SwitchArgument tsvArg = new SwitchArgument('y', "tsv", "Makes Snaffler output as tsv.", false); + SwitchArgument dfsArg = new SwitchArgument('f', "dfs", "Limits Snaffler to finding file shares via DFS, for \"OPSEC\" reasons.", false); + SwitchArgument findSharesOnlyArg = new SwitchArgument('a', "sharesonly", + "Stops after finding shares, doesn't walk their filesystems.", false); + ValueArgument compExclusionArg = new ValueArgument('k', "exclusions", "Path to a file containing a list of computers to exclude from scanning."); + ValueArgument compTargetArg = new ValueArgument('n', "comptarget", "List of computers in a file(e.g C:\targets.txt), a single Computer (or comma separated list) to target."); + ValueArgument ruleDirArg = new ValueArgument('p', "rulespath", "Path to a directory full of toml-formatted rules. Snaffler will load all of these in place of the default ruleset."); + ValueArgument logType = new ValueArgument('t', "logtype", "Type of log you would like to output. Currently supported options are plain and JSON. Defaults to plain."); + ValueArgument timeOutArg = new ValueArgument('e', "timeout", + "Interval between status updates (in minutes) also acts as a timeout for AD data to be gathered via LDAP. Turn this knob up if you aren't getting any computers from AD when you run Snaffler through a proxy or other slow link. Default = 5"); + // list of letters i haven't used yet: gnqw + + CommandLineParser.CommandLineParser parser = new CommandLineParser.CommandLineParser(); + parser.Arguments.Add(timeOutArg); + parser.Arguments.Add(configFileArg); + parser.Arguments.Add(outFileArg); + parser.Arguments.Add(helpArg); + parser.Arguments.Add(stdOutArg); + parser.Arguments.Add(snaffleArg); + parser.Arguments.Add(snaffleSizeArg); + parser.Arguments.Add(dirTargetArg); + parser.Arguments.Add(interestLevel); + parser.Arguments.Add(domainArg); + parser.Arguments.Add(verboseArg); + parser.Arguments.Add(domainControllerArg); + parser.Arguments.Add(maxGrepSizeArg); + parser.Arguments.Add(grepContextArg); + parser.Arguments.Add(domainUserArg); + parser.Arguments.Add(tsvArg); + parser.Arguments.Add(dfsArg); + parser.Arguments.Add(findSharesOnlyArg); + parser.Arguments.Add(maxThreadsArg); + parser.Arguments.Add(compTargetArg); + parser.Arguments.Add(ruleDirArg); + parser.Arguments.Add(logType); + parser.Arguments.Add(compExclusionArg); + + // extra check to handle builtin behaviour from cmd line arg parser + if ((args.Contains("--help") || args.Contains("/?") || args.Contains("help") || args.Contains("-h") || args.Length == 0)) + { + parser.ShowUsage(); + return null; + } + + + try + { + parser.ParseCommandLine(args); + + if (timeOutArg.Parsed && !String.IsNullOrWhiteSpace(timeOutArg.Value)) + { + int timeOutVal; + if (int.TryParse(timeOutArg.Value, out timeOutVal)) + { + Mq.Info("Set timeout/update interval to " + timeOutVal.ToString() + " minutes."); + parsedConfig.TimeOut = timeOutVal; + } + else + { + Mq.Error("Invalid timeout value passed, defaulting to 5 mins."); + } + } + + if (logType.Parsed && !String.IsNullOrWhiteSpace(logType.Value)) + { + //Set the default to plain + parsedConfig.LogType = LogType.Plain; + //if they set a different type then replace it with the new type. + if (logType.Value.ToLower() == "json") + { + parsedConfig.LogType = LogType.JSON; + } + else + { + Mq.Info("Invalid type argument passed (" + logType.Value + ") defaulting to plaintext"); + } + } + + if (ruleDirArg.Parsed && !String.IsNullOrWhiteSpace(ruleDirArg.Value)) + { + parsedConfig.RuleDir = ruleDirArg.Value; + } + + // get the args into our config + + // output args + if (outFileArg.Parsed && (!String.IsNullOrEmpty(outFileArg.Value))) + { + parsedConfig.LogToFile = true; + parsedConfig.LogFilePath = outFileArg.Value; + Mq.Degub("Logging to file at " + parsedConfig.LogFilePath); + } + + if (dfsArg.Parsed) + { + parsedConfig.DfsOnly = dfsArg.Value; + } + if (compExclusionArg.Parsed) + { + List compExclusions = new List(); + string[] fileLines = await File.ReadAllLinesAsync(compExclusionArg.Value); + foreach (string line in fileLines) + { + if (isIP(line)) + { + compExclusions.Add(line); + } + else + { + try + { + IPHostEntry result = await Dns.GetHostEntryAsync(line); + foreach (IPAddress ipAddress in result.AddressList) + { + compExclusions.Add(ipAddress.ToString()); + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + continue; + } + } + } + if (compExclusions.Count > 0) + { + parsedConfig.ComputerExclusions = compExclusions; + parsedConfig.ComputerExclusionFile = compExclusionArg.Value; + } + else + { + throw new Exception("Failed to get a valid list of excluded computers from the excluded computers list."); + } + } + + if (compTargetArg.Parsed) + { + List compTargets = new List(); + if (compTargetArg.Value.Contains(Path.DirectorySeparatorChar)) + { + string[] targetLines = await File.ReadAllLinesAsync(compTargetArg.Value); + compTargets.AddRange(targetLines.Select(line => line.Trim())); + } + else if (compTargetArg.Value.Contains(",")) + { + compTargets.AddRange(compTargetArg.Value.Split(',').Select(x => x.Trim())); + } + else + { + compTargets.Add(compTargetArg.Value.Trim()); + } + parsedConfig.ComputerTargets = compTargets.ToArray(); + } + + if (findSharesOnlyArg.Parsed) + { + parsedConfig.ScanFoundShares = false; + } + if (maxThreadsArg.Parsed) + { + parsedConfig.MaxThreads = maxThreadsArg.Value; + } + + parsedConfig.ShareThreads = parsedConfig.MaxThreads / 3; + parsedConfig.FileThreads = parsedConfig.MaxThreads / 3; + parsedConfig.TreeThreads = parsedConfig.MaxThreads / 3; + + if (tsvArg.Parsed) + { + parsedConfig.LogTSV = true; + if (parsedConfig.Separator == ' ') + { + parsedConfig.Separator = '\t'; + } + } + + // Set loglevel. + if (verboseArg.Parsed) + { + parsedConfig.LogLevelString = verboseArg.Value; + Mq.Degub("Requested verbosity level: " + parsedConfig.LogLevelString); + } + + // if enabled, display findings to the console + parsedConfig.LogToConsole = stdOutArg.Parsed; + Mq.Degub("Enabled logging to stdout."); + + // args that tell us about targeting + if ((domainArg.Parsed) && (!String.IsNullOrEmpty(domainArg.Value))) + { + parsedConfig.TargetDomain = domainArg.Value; + Mq.Degub("Target domain is " + domainArg.Value); + } + + if ((domainControllerArg.Parsed) && (!String.IsNullOrEmpty(domainControllerArg.Value))) + { + parsedConfig.TargetDc = domainControllerArg.Value; + Mq.Degub("Target DC is " + domainControllerArg.Value); + } + + if (domainUserArg.Parsed) + { + parsedConfig.DomainUserRules = true; + Mq.Degub("Enabled use of domain user accounts in rules."); + } + + if (dirTargetArg.Parsed) + { + parsedConfig.ShareFinderEnabled = false; + //Console.WriteLine(dirTargetArg.Value); + string pathTarget = dirTargetArg.Value; + if (dirTargetArg.Value.Length > 4) + { + pathTarget = dirTargetArg.Value.TrimEnd('\\'); + } + parsedConfig.PathTargets.Add(pathTarget); + //Console.WriteLine(parsedConfig.PathTargets[0]); + Mq.Degub("Disabled finding shares."); + Mq.Degub("Target path is " + dirTargetArg.Value); + } + + if (maxGrepSizeArg.Parsed) + { + parsedConfig.MaxSizeToGrep = maxGrepSizeArg.Value; + Mq.Degub("We won't bother looking inside files if they're bigger than " + parsedConfig.MaxSizeToGrep + + " bytes"); + } + + if (snaffleSizeArg.Parsed) + { + parsedConfig.MaxSizeToSnaffle = snaffleSizeArg.Value; + } + + if (interestLevel.Parsed) + { + parsedConfig.InterestLevel = interestLevel.Value; + Mq.Degub("Requested interest level: " + parsedConfig.InterestLevel); + } + + // how many bytes + if (grepContextArg.Parsed) + { + parsedConfig.MatchContextBytes = grepContextArg.Value; + Mq.Degub( + "We'll show you " + grepContextArg.Value + + " bytes of context around matches inside files."); + } + + // if enabled, grab a copy of files that we like. + if (snaffleArg.Parsed) + { + if (snaffleArg.Value.Length <= 0) + { + Mq.Error("-m or -mirror arg requires a path value."); + throw new ArgumentException("Invalid argument combination."); + } + + parsedConfig.Snaffle = true; + parsedConfig.SnafflePath = snaffleArg.Value.TrimEnd('\\'); + Mq.Degub("Mirroring matched files to path " + parsedConfig.SnafflePath); + } + + if (configFileArg.Parsed) + { + if (configFileArg.Value.Equals("generate")) + { + using var result = CsTomlSerializer.Serialize(parsedConfig); + File.WriteAllBytes(".\\default.toml", result.ByteSpan.ToArray()); + Console.WriteLine("Wrote config values to .\\default.toml"); + parsedConfig.LogToConsole = true; + Mq.Degub("Enabled logging to stdout."); + return null; + } + else + { + string configFile = configFileArg.Value; + parsedConfig = CsTomlSerializer.Deserialize(await File.ReadAllBytesAsync(configFile)); + Mq.Info("Read config file from " + configFile); + } + } + + if (!parsedConfig.LogToConsole && !parsedConfig.LogToFile) + { + Mq.Error( + "\nYou didn't enable output to file or to the console so you won't see any results or debugs or anything. Your l0ss."); + throw new ArgumentException("Pointless argument combination."); + } + + if (parsedConfig.ClassifierRules.Count <= 0) + { + if (String.IsNullOrWhiteSpace(parsedConfig.RuleDir)) + { + // get all the embedded toml file resources + string[] resourceNames = Assembly.GetExecutingAssembly().GetManifestResourceNames(); + StringBuilder sb = new StringBuilder(); + + foreach (string resourceName in resourceNames) + { + if (!resourceName.EndsWith(".toml")) + { + // skip this one as it's just metadata + continue; + } + string ruleFile = ReadResource(resourceName); + sb.AppendLine(ruleFile); + } + + string bulktoml = sb.ToString(); + + // deserialise the toml to an actual ruleset + RuleSet ruleSet = CsTomlSerializer.Deserialize(System.Text.Encoding.UTF8.GetBytes(bulktoml)); + + // stick the rules in our config! + parsedConfig.ClassifierRules = ruleSet.ClassifierRules; + } + else + { + string[] tomlfiles = Directory.GetFiles(parsedConfig.RuleDir, "*.toml", SearchOption.AllDirectories); + StringBuilder sb = new StringBuilder(); + foreach (string tomlfile in tomlfiles) + { + string tomlstring = await File.ReadAllTextAsync(tomlfile); + sb.AppendLine(tomlstring); + } + string bulktoml = sb.ToString(); + // deserialise the toml to an actual ruleset + RuleSet ruleSet = CsTomlSerializer.Deserialize(System.Text.Encoding.UTF8.GetBytes(bulktoml)); + + // stick the rules in our config! + parsedConfig.ClassifierRules = ruleSet.ClassifierRules; + } + } + + parsedConfig.PrepareClassifiers(); + } + catch (Exception e) + { + Mq.Error(e.ToString()); + throw; + } + + return parsedConfig; + } + + + + } +} diff --git a/Snaffler/FodyWeavers.xml b/Snaffler/FodyWeavers.xml deleted file mode 100644 index f1dea8fc..00000000 --- a/Snaffler/FodyWeavers.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/Snaffler/FodyWeavers.xsd b/Snaffler/FodyWeavers.xsd deleted file mode 100644 index 8ac6e927..00000000 --- a/Snaffler/FodyWeavers.xsd +++ /dev/null @@ -1,111 +0,0 @@ - - - - - - - - - - - - A list of assembly names to exclude from the default action of "embed all Copy Local references", delimited with line breaks - - - - - A list of assembly names to include from the default action of "embed all Copy Local references", delimited with line breaks. - - - - - A list of unmanaged 32 bit assembly names to include, delimited with line breaks. - - - - - A list of unmanaged 64 bit assembly names to include, delimited with line breaks. - - - - - The order of preloaded assemblies, delimited with line breaks. - - - - - - This will copy embedded files to disk before loading them into memory. This is helpful for some scenarios that expected an assembly to be loaded from a physical file. - - - - - Controls if .pdbs for reference assemblies are also embedded. - - - - - Embedded assemblies are compressed by default, and uncompressed when they are loaded. You can turn compression off with this option. - - - - - As part of Costura, embedded assemblies are no longer included as part of the build. This cleanup can be turned off. - - - - - Costura by default will load as part of the module initialization. This flag disables that behavior. Make sure you call CosturaUtility.Initialize() somewhere in your code. - - - - - Costura will by default use assemblies with a name like 'resources.dll' as a satellite resource and prepend the output path. This flag disables that behavior. - - - - - A list of assembly names to exclude from the default action of "embed all Copy Local references", delimited with | - - - - - A list of assembly names to include from the default action of "embed all Copy Local references", delimited with |. - - - - - A list of unmanaged 32 bit assembly names to include, delimited with |. - - - - - A list of unmanaged 64 bit assembly names to include, delimited with |. - - - - - The order of preloaded assemblies, delimited with |. - - - - - - - - 'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed. - - - - - A comma-separated list of error codes that can be safely ignored in assembly verification. - - - - - 'false' to turn off automatic generation of the XML Schema file. - - - - - \ No newline at end of file diff --git a/Snaffler/Logger.cs b/Snaffler/Logger.cs new file mode 100644 index 00000000..04637c86 --- /dev/null +++ b/Snaffler/Logger.cs @@ -0,0 +1,216 @@ +using System; +using System.IO; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; +using SnaffCore.Config; + +#nullable enable +namespace Snaffler +{ + public enum LogLevel + { + Trace = 0, + Debug = 1, + Info = 2, + Warn = 3, + Error = 4, + Fatal = 5 + } + + public static class Logger + { + private static readonly object _lock = new object(); + private static StreamWriter? _fileWriter; + private static bool _consoleEnabled; + private static bool _fileEnabled; + private static LogLevel _minLevel; + private static LogType _logType; + private static char _separator; + private static readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + public static void Configure(bool logToConsole, bool logToFile, string? filePath, LogLevel minLevel, LogType logType, char separator) + { + _consoleEnabled = logToConsole; + _fileEnabled = logToFile && !string.IsNullOrWhiteSpace(filePath); + _minLevel = minLevel; + _logType = logType; + _separator = separator; + if (_fileEnabled) + { + _fileWriter = new StreamWriter(filePath!, append: false, System.Text.Encoding.UTF8) + { + AutoFlush = true + }; + } + } + + public static void Close() + { + lock (_lock) + { + if (_fileWriter != null) + { + _fileWriter.Flush(); + _fileWriter.Dispose(); + } + _fileWriter = null; + } + } + + private static void WriteHighlighted(string text) + { + if (text.IndexOfAny(new[] { '{', '<', '(' }) == -1) + { + Console.Write(text); + return; + } + var oldFg = Console.ForegroundColor; + var oldBg = Console.BackgroundColor; + bool curlyDone = false; + bool angleDone = false; + bool parenDone = false; + for (int i = 0; i < text.Length; i++) + { + char c = text[i]; + if (!curlyDone && c == '{') + { + int end = text.IndexOf('}', i); + if (end > i) + { + string token = text.Substring(i, end - i + 1); + Console.ForegroundColor = token switch + { + "{Green}" => ConsoleColor.DarkGreen, + "{Yellow}" => ConsoleColor.DarkYellow, + "{Red}" => ConsoleColor.DarkRed, + "{Black}" => ConsoleColor.Black, + _ => oldFg + }; + Console.BackgroundColor = ConsoleColor.White; + Console.Write(token); + Console.ForegroundColor = oldFg; + Console.BackgroundColor = oldBg; + i = end; + curlyDone = true; + continue; + } + } + else if (!angleDone && c == '<') + { + int end = text.IndexOf('>', i); + if (end > i) + { + Console.ForegroundColor = ConsoleColor.Cyan; + Console.BackgroundColor = ConsoleColor.Black; + Console.Write(text.Substring(i, end - i + 1)); + Console.ForegroundColor = oldFg; + Console.BackgroundColor = oldBg; + i = end; + angleDone = true; + continue; + } + } + else if (!parenDone && c == '(') + { + int end = text.IndexOf(')', i); + if (end > i) + { + Console.ForegroundColor = ConsoleColor.DarkMagenta; + Console.BackgroundColor = ConsoleColor.Black; + Console.Write(text.Substring(i, end - i + 1)); + Console.ForegroundColor = oldFg; + Console.BackgroundColor = oldBg; + i = end; + parenDone = true; + continue; + } + } + + Console.ForegroundColor = oldFg; + Console.BackgroundColor = oldBg; + Console.Write(c); + } + Console.ForegroundColor = oldFg; + Console.BackgroundColor = oldBg; + } + + private static void WriteConsoleInternal(string prefix, string tag, ConsoleColor tagColor, string message) + { + lock (_lock) + { + var oldFg = Console.ForegroundColor; + var oldBg = Console.BackgroundColor; + Console.ForegroundColor = ConsoleColor.DarkGray; + Console.BackgroundColor = ConsoleColor.Black; + Console.Write(prefix); + Console.ForegroundColor = tagColor; + Console.BackgroundColor = ConsoleColor.Black; + Console.Write(tag); + Console.ForegroundColor = oldFg; + Console.BackgroundColor = oldBg; + Console.Write(_separator); + WriteHighlighted(message); + Console.WriteLine(); + } + } + + private static void WriteConsole(string prefix, string tag, ConsoleColor tagColor, string message) + { + WriteConsoleInternal(prefix, tag, tagColor, message); + } + + private static void WriteFile(string text) + { + lock (_lock) + { + if (_fileWriter == null) return; + _fileWriter.WriteLine(text); + _fileWriter.Flush(); + } + } + + private static string BuildJson(LogLevel level, DateTime time, string message, object? data) + { + var obj = new + { + time = time.ToUniversalTime().ToString("O"), + level = level.ToString(), + message, + eventProperties = data + }; + return JsonSerializer.Serialize(obj, _jsonOptions); + } + + private static void Log(LogLevel level, DateTime time, string prefix, string tag, ConsoleColor tagColor, string message, object? data = null) + { + if (level < _minLevel) return; + + string plain = string.Concat(prefix, tag, _separator, message); + string output = _logType == LogType.JSON ? BuildJson(level, time, plain, data) : plain; + + if (_consoleEnabled) + { + WriteConsole(prefix, tag, tagColor, message); + } + if (_fileEnabled) + { + WriteFile(output); + } + } + + public static void Trace(DateTime time, string prefix, string message, object? data = null) => Log(LogLevel.Trace, time, prefix, "[Trace]", ConsoleColor.DarkGray, message, data); + public static void Debug(DateTime time, string prefix, string message, object? data = null) => Log(LogLevel.Debug, time, prefix, "[Debug]", ConsoleColor.Gray, message, data); + public static void Info(DateTime time, string prefix, string message, object? data = null) => Log(LogLevel.Info, time, prefix, "[Info]", ConsoleColor.White, message, data); + public static void Warn(DateTime time, string prefix, string message, object? data = null) => Log(LogLevel.Warn, time, prefix, "[Warn]", ConsoleColor.Yellow, message, data); + public static void Error(DateTime time, string prefix, string message, object? data = null) => Log(LogLevel.Error, time, prefix, "[Error]", ConsoleColor.Magenta, message, data); + public static void Fatal(DateTime time, string prefix, string message, object? data = null) => Log(LogLevel.Fatal, time, prefix, "[Fatal]", ConsoleColor.Red, message, data); + + public static void File(DateTime time, string prefix, string message, object? data = null) => Log(LogLevel.Warn, time, prefix, "[File]", ConsoleColor.Green, message, data); + public static void Share(DateTime time, string prefix, string message, object? data = null) => Log(LogLevel.Warn, time, prefix, "[Share]", ConsoleColor.Yellow, message, data); + public static void Dir(DateTime time, string prefix, string message, object? data = null) => Log(LogLevel.Warn, time, prefix, "[Dir]", ConsoleColor.Green, message, data); + } +} diff --git a/Snaffler/RuleSet.cs b/Snaffler/RuleSet.cs index 0b867c3c..a7ba33fa 100644 --- a/Snaffler/RuleSet.cs +++ b/Snaffler/RuleSet.cs @@ -1,10 +1,13 @@ using System.Collections.Generic; using SnaffCore.Classifiers; +using CsToml; -namespace Snaffler +namespace Snaffler { - public class RuleSet + [TomlSerializedObject] + public partial class RuleSet { + [TomlValueOnSerialized] public List ClassifierRules { get; set; } = new List(); } } \ No newline at end of file diff --git a/Snaffler/SnaffleRunner.cs b/Snaffler/SnaffleRunner.cs index 0917aea1..43c4a447 100644 --- a/Snaffler/SnaffleRunner.cs +++ b/Snaffler/SnaffleRunner.cs @@ -1,7 +1,3 @@ -using NLog; -using NLog.Config; -using NLog.Layouts; -using NLog.Targets; using SnaffCore; using SnaffCore.Concurrency; using SnaffCore.Config; @@ -9,7 +5,6 @@ using System.Diagnostics; using System.IO; using System.Threading.Tasks; -using System.Threading.Tasks; using System.Text.RegularExpressions; using System.Threading; @@ -17,7 +12,6 @@ namespace Snaffler { public class SnaffleRunner { - private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); private BlockingMq Mq { get; set; } private LogLevel LogLevel { get; set; } private Options Options { get; set; } @@ -77,119 +71,18 @@ public async Task RunAsync(string[] args) dirResultTemplate = "{{{0}}}({1})"; } //------------------------------------------ - // set up new fangled logging + // set up logging //------------------------------------------ - LoggingConfiguration nlogConfig = new LoggingConfiguration(); - nlogConfig.Variables["encoding"] = "utf8"; - ColoredConsoleTarget logconsole = null; - FileTarget logfile = null; ParseLogLevelString(Options.LogLevelString); - // Targets where to log to: File and Console - if (Options.LogToConsole) - { - logconsole = new ColoredConsoleTarget("logconsole") - { - DetectOutputRedirected = true, - UseDefaultRowHighlightingRules = false, - WordHighlightingRules = - { - new ConsoleWordHighlightingRule("{Green}", ConsoleOutputColor.DarkGreen, - ConsoleOutputColor.White), - new ConsoleWordHighlightingRule("{Yellow}", ConsoleOutputColor.DarkYellow, - ConsoleOutputColor.White), - new ConsoleWordHighlightingRule("{Red}", ConsoleOutputColor.DarkRed, - ConsoleOutputColor.White), - new ConsoleWordHighlightingRule("{Black}", ConsoleOutputColor.Black, - ConsoleOutputColor.White), - - new ConsoleWordHighlightingRule("[Trace]", ConsoleOutputColor.DarkGray, - ConsoleOutputColor.Black), - new ConsoleWordHighlightingRule("[Degub]", ConsoleOutputColor.Gray, - ConsoleOutputColor.Black), - new ConsoleWordHighlightingRule("[Info]", ConsoleOutputColor.White, - ConsoleOutputColor.Black), - new ConsoleWordHighlightingRule("[Error]", ConsoleOutputColor.Magenta, - ConsoleOutputColor.Black), - new ConsoleWordHighlightingRule("[Fatal]", ConsoleOutputColor.Red, - ConsoleOutputColor.Black), - new ConsoleWordHighlightingRule("[File]", ConsoleOutputColor.Green, - ConsoleOutputColor.Black), - new ConsoleWordHighlightingRule("[Share]", ConsoleOutputColor.Yellow, - ConsoleOutputColor.Black), - new ConsoleWordHighlightingRule - { - CompileRegex = true, - Regex = @"<.*\|.*\|.*\|.*?>", - ForegroundColor = ConsoleOutputColor.Cyan, - BackgroundColor = ConsoleOutputColor.Black - }, - new ConsoleWordHighlightingRule - { - CompileRegex = true, - Regex = @"^\d\d\d\d-\d\d\-\d\d \d\d:\d\d:\d\d [\+-]\d\d:\d\d ", - ForegroundColor = ConsoleOutputColor.DarkGray, - BackgroundColor = ConsoleOutputColor.Black - }, - new ConsoleWordHighlightingRule - { - CompileRegex = true, - Regex = @"\((?:[^\)]*\)){1}", - ForegroundColor = ConsoleOutputColor.DarkMagenta, - BackgroundColor = ConsoleOutputColor.Black - } - } - }; - if (LogLevel == LogLevel.Warn) - { - nlogConfig.AddRule(LogLevel.Warn, LogLevel.Warn, logconsole); - } - else - { - nlogConfig.AddRule(LogLevel, LogLevel.Fatal, logconsole); - } - logconsole.Layout = "${message}"; - } - - if (Options.LogToFile) - { - logfile = new FileTarget("logfile") { FileName = Options.LogFilePath }; - if (LogLevel == LogLevel.Warn) - { - nlogConfig.AddRule(LogLevel.Warn, LogLevel.Warn, logfile); - } - else - { - nlogConfig.AddRule(LogLevel, LogLevel.Fatal, logfile); - } - if (Options.LogType == LogType.Plain) - { - logfile.Layout = "${message}"; - } - else if (Options.LogType == LogType.JSON) - { - var eventProperties = new JsonLayout(); - eventProperties.IncludeAllProperties = true; - eventProperties.MaxRecursionLimit = 5; - var jsonLayout = new JsonLayout - { - Attributes = - { - new JsonAttribute("time", "${longdate}"), - new JsonAttribute("level", "${level}"), - new JsonAttribute("message", "${message}"), - new JsonAttribute("eventProperties", eventProperties, - //don't escape layout - false) - } - }; - logfile.Layout = jsonLayout; - } - } - - // Apply config - LogManager.Configuration = nlogConfig; + Logger.Configure( + Options.LogToConsole, + Options.LogToFile, + Options.LogFilePath, + LogLevel, + Options.LogType, + Options.Separator); //------------------------------------------- @@ -203,6 +96,7 @@ public async Task RunAsync(string[] args) Task controllerTask = Task.Run(() => controller.ExecuteAsync()); await HandleOutputAsync(); await controllerTask; + Logger.Close(); return; } catch (Exception e) @@ -251,41 +145,41 @@ private async Task HandleOutputAsync() private void ProcessMessage(SnafflerMessage message) { - // standardized time formatting, UTC - string datetime = String.Format("{1}{0}{2:u}{0}", Options.Separator, hostString(), message.DateTime.ToUniversalTime()); + string prefix = hostString() + Options.Separator + + message.DateTime.ToUniversalTime().ToString("u") + Options.Separator; switch (message.Type) { case SnafflerMessageType.Trace: - Logger.Trace(datetime + "[Trace]" + Options.Separator + message.Message); + Logger.Trace(message.DateTime, prefix, message.Message); break; case SnafflerMessageType.Degub: - Logger.Debug(datetime + "[Degub]" + Options.Separator + message.Message); + Logger.Debug(message.DateTime, prefix, message.Message); break; case SnafflerMessageType.Info: - Logger.Info(datetime + "[Info]" + Options.Separator + message.Message); + Logger.Info(message.DateTime, prefix, message.Message); break; case SnafflerMessageType.FileResult: - Logger.Warn(datetime + "[File]" + Options.Separator + FileResultLogFromMessage(message)); + Logger.File(message.DateTime, prefix, FileResultLogFromMessage(message)); break; case SnafflerMessageType.DirResult: - Logger.Warn(datetime + "[Dir]" + Options.Separator + DirResultLogFromMessage(message)); + Logger.Dir(message.DateTime, prefix, DirResultLogFromMessage(message)); break; case SnafflerMessageType.ShareResult: - Logger.Warn(datetime + "[Share]" + Options.Separator + ShareResultLogFromMessage(message)); + Logger.Share(message.DateTime, prefix, ShareResultLogFromMessage(message)); break; case SnafflerMessageType.Error: - Logger.Error(datetime + "[Error]" + Options.Separator + message.Message); + Logger.Error(message.DateTime, prefix, message.Message); break; case SnafflerMessageType.Fatal: - Logger.Fatal(datetime + "[Fatal]" + Options.Separator + message.Message); + Logger.Fatal(message.DateTime, prefix, message.Message); if (Debugger.IsAttached) { Console.ReadKey(); } break; case SnafflerMessageType.Finish: - Logger.Info("Snaffler out."); + Logger.Info(message.DateTime, prefix, "Snaffler out."); if (Debugger.IsAttached) { @@ -298,49 +192,40 @@ private void ProcessMessage(SnafflerMessage message) private async Task ProcessMessageJSONAsync(SnafflerMessage message) { - // standardized time formatting, UTC - string datetime = String.Format("{1}{0}{2:u}{0}", Options.Separator, hostString(), message.DateTime.ToUniversalTime()); + string prefix = hostString() + Options.Separator; switch (message.Type) { case SnafflerMessageType.Trace: - //Logger.Trace(message); - Logger.Trace(datetime + "[Trace]" + Options.Separator + message.Message, message); + Logger.Trace(message.DateTime, prefix, message.Message, message); break; case SnafflerMessageType.Degub: - //Logger.Debug(message); - Logger.Debug(datetime + "[Degub]" + Options.Separator + message.Message, message); + Logger.Debug(message.DateTime, prefix, message.Message, message); break; case SnafflerMessageType.Info: - //Logger.Info(message); - Logger.Info(datetime + "[Info]" + Options.Separator + message.Message, message); + Logger.Info(message.DateTime, prefix, message.Message, message); break; case SnafflerMessageType.FileResult: - //Logger.Warn(message); - Logger.Warn(datetime + "[File]" + Options.Separator + FileResultLogFromMessage(message), message); + Logger.File(message.DateTime, prefix, FileResultLogFromMessage(message), message); break; case SnafflerMessageType.DirResult: - //Logger.Warn(message); - Logger.Warn(datetime + "[Dir]" + Options.Separator + DirResultLogFromMessage(message), message); + Logger.Dir(message.DateTime, prefix, DirResultLogFromMessage(message), message); break; case SnafflerMessageType.ShareResult: - //Logger.Warn(message); - Logger.Warn(datetime + "[Share]" + Options.Separator + ShareResultLogFromMessage(message), message); + Logger.Share(message.DateTime, prefix, ShareResultLogFromMessage(message), message); break; case SnafflerMessageType.Error: - //Logger.Error(message); - Logger.Error(datetime + "[Error]" + Options.Separator + message.Message, message); + Logger.Error(message.DateTime, prefix, message.Message, message); break; case SnafflerMessageType.Fatal: - //Logger.Fatal(message); - Logger.Fatal(datetime + "[Fatal]" + Options.Separator + message.Message, message); + Logger.Fatal(message.DateTime, prefix, message.Message, message); if (Debugger.IsAttached) { Console.ReadKey(); } break; case SnafflerMessageType.Finish: - Logger.Info("Snaffler out."); + Logger.Info(message.DateTime, prefix, "Snaffler out."); if (Debugger.IsAttached) { @@ -349,7 +234,7 @@ private async Task ProcessMessageJSONAsync(SnafflerMessage message) } if (Options.LogType == LogType.JSON) { - Logger.Info("Normalising output, please wait..."); + Logger.Info(message.DateTime, prefix, "Normalising output, please wait..."); await FixJSONOutputAsync(); } break; @@ -493,19 +378,27 @@ private static String BytesToString(long byteCount) public void WriteColor(string textToWrite, ConsoleColor fgColor) { - Console.ForegroundColor = fgColor; + if (Console.IsOutputRedirected) + { + Console.Write(textToWrite); + return; + } + Console.ForegroundColor = fgColor; Console.Write(textToWrite); - Console.ResetColor(); } public void WriteColorLine(string textToWrite, ConsoleColor fgColor) { - Console.ForegroundColor = fgColor; + if (Console.IsOutputRedirected) + { + Console.WriteLine(textToWrite); + return; + } + Console.ForegroundColor = fgColor; Console.WriteLine(textToWrite); - Console.ResetColor(); } diff --git a/Snaffler/Snaffler.csproj b/Snaffler/Snaffler.csproj index db626596..73ac66c7 100644 --- a/Snaffler/Snaffler.csproj +++ b/Snaffler/Snaffler.csproj @@ -5,6 +5,7 @@ false Snaffler Snaffler + true @@ -16,7 +17,7 @@ - - + + From 434d5c2b14549639c553db9811b5a93e90c7ee7c Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 9 Jun 2025 11:00:51 +0900 Subject: [PATCH 07/10] Use channel for async logging --- Snaffler/Logger.cs | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/Snaffler/Logger.cs b/Snaffler/Logger.cs index 04637c86..9c3795c2 100644 --- a/Snaffler/Logger.cs +++ b/Snaffler/Logger.cs @@ -1,6 +1,8 @@ using System; using System.IO; using System.Collections.Generic; +using System.Threading.Channels; +using System.Threading.Tasks; using System.Text.Json; using System.Text.Json.Serialization; using SnaffCore.Config; @@ -27,11 +29,15 @@ public static class Logger private static LogLevel _minLevel; private static LogType _logType; private static char _separator; + private static Channel? _channel; + private static Task? _consumerTask; private static readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; + private readonly record struct LogEntry(string Prefix, string Tag, ConsoleColor TagColor, string Message, string FileMessage); + public static void Configure(bool logToConsole, bool logToFile, string? filePath, LogLevel minLevel, LogType logType, char separator) { _consoleEnabled = logToConsole; @@ -46,10 +52,14 @@ public static void Configure(bool logToConsole, bool logToFile, string? filePath AutoFlush = true }; } + _channel = Channel.CreateUnbounded(); + _consumerTask = Task.Run(ProcessQueueAsync); } public static void Close() { + _channel?.Writer.TryComplete(); + _consumerTask?.Wait(); lock (_lock) { if (_fileWriter != null) @@ -173,6 +183,22 @@ private static void WriteFile(string text) } } + private static async Task ProcessQueueAsync() + { + if (_channel == null) return; + await foreach (var entry in _channel.Reader.ReadAllAsync()) + { + if (_consoleEnabled) + { + WriteConsoleInternal(entry.Prefix, entry.Tag, entry.TagColor, entry.Message); + } + if (_fileEnabled) + { + WriteFile(entry.FileMessage); + } + } + } + private static string BuildJson(LogLevel level, DateTime time, string message, object? data) { var obj = new @@ -191,14 +217,21 @@ private static void Log(LogLevel level, DateTime time, string prefix, string tag string plain = string.Concat(prefix, tag, _separator, message); string output = _logType == LogType.JSON ? BuildJson(level, time, plain, data) : plain; - - if (_consoleEnabled) + var entry = new LogEntry(prefix, tag, tagColor, message, output); + if (_channel != null) { - WriteConsole(prefix, tag, tagColor, message); + _channel.Writer.TryWrite(entry); } - if (_fileEnabled) + else { - WriteFile(output); + if (_consoleEnabled) + { + WriteConsoleInternal(prefix, tag, tagColor, message); + } + if (_fileEnabled) + { + WriteFile(output); + } } } From afdad853a706fcac197d09be1da6ef73f862dadc Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 9 Jun 2025 12:12:03 +1000 Subject: [PATCH 08/10] Revert README.md --- README.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/README.md b/README.md index ed3d45e6..490b4c0e 100644 --- a/README.md +++ b/README.md @@ -275,17 +275,6 @@ The solution was UltraSnaffler, which is just a second `.sln` file that enables WARNING: Snaffler's default rules don't include any that will look inside Office docs or PDFs, because we found it really difficult to write any that weren't going to just take *years* to finish a run in a typical corporate environment. *Be warned, looking inside these docs is a lot slower than looking inside good old fashioned text files, and a typical environment will have an absolute mountain of low-value Office docs and PDFs.* -## Building Native AOT executables - -Snaffler and its library projects now support publishing as native executables. -To create a self‑contained binary, run a command like: - -```bash -dotnet publish Snaffler/Snaffler.csproj -c Release -r win-x64 --self-contained -``` - -Replace the runtime identifier with the target OS and architecture of your choice. - ## How does the config file thing work? This is actually really neat IMO. From 1053a7e09a528c253ec5f224ee5bb357357fd6a8 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Sun, 15 Jun 2025 11:07:19 +0900 Subject: [PATCH 09/10] Retarget Windows profile and cleanup --- .gitignore | 1 + SnaffCore/Concurrency/BlockingMq.cs | 26 ++++++++- SnaffCore/Config/CsTomlStub.cs | 11 ++++ SnaffCore/SnaffCore.csproj | 12 ++++- SnaffCore/UltraSnaffCore.csproj | 14 ++++- Snaffler/Config.cs | 18 +++---- Snaffler/FileCompat.cs | 45 ++++++++++++++++ Snaffler/FodyWeavers.xml | 4 ++ Snaffler/IsExternalInit.cs | 6 +++ Snaffler/Logger.cs | 52 +++++++++++++++++- .../Properties/PublishProfiles/net451.pubxml | 11 ++++ .../Properties/PublishProfiles/net9.pubxml | 9 ++++ Snaffler/SnaffleRunner.cs | 11 +++- Snaffler/Snaffler.csproj | 20 ++++++- Snaffler/TomlCompat.cs | 53 +++++++++++++++++++ publish.bat | 9 ++++ 16 files changed, 283 insertions(+), 19 deletions(-) create mode 100644 SnaffCore/Config/CsTomlStub.cs create mode 100644 Snaffler/FileCompat.cs create mode 100644 Snaffler/FodyWeavers.xml create mode 100644 Snaffler/IsExternalInit.cs create mode 100644 Snaffler/Properties/PublishProfiles/net451.pubxml create mode 100644 Snaffler/Properties/PublishProfiles/net9.pubxml create mode 100644 Snaffler/TomlCompat.cs create mode 100644 publish.bat diff --git a/.gitignore b/.gitignore index 2bccbac9..4a57d242 100644 --- a/.gitignore +++ b/.gitignore @@ -181,6 +181,7 @@ publish/ # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted *.pubxml +!Snaffler/Properties/PublishProfiles/*.pubxml *.publishproj # Microsoft Azure Web App publish settings. Comment the next line if you want to diff --git a/SnaffCore/Concurrency/BlockingMq.cs b/SnaffCore/Concurrency/BlockingMq.cs index c47cb89b..2a3830c6 100644 --- a/SnaffCore/Concurrency/BlockingMq.cs +++ b/SnaffCore/Concurrency/BlockingMq.cs @@ -1,7 +1,10 @@ using SnaffCore.Classifiers; using System; using System.Collections.Concurrent; +#if NETFRAMEWORK +#else using System.Threading.Channels; +#endif namespace SnaffCore.Concurrency { @@ -20,17 +23,30 @@ public static BlockingMq GetMq() } // Message Queue +#if NETFRAMEWORK + private BlockingCollection _collection; + public BlockingCollection Collection => _collection; +#else private Channel _channel; public ChannelReader Reader => _channel.Reader; +#endif private BlockingMq() { +#if NETFRAMEWORK + _collection = new BlockingCollection(); +#else _channel = Channel.CreateUnbounded(); +#endif } private void Enqueue(SnafflerMessage message) { +#if NETFRAMEWORK + _collection.Add(message); +#else _channel.Writer.TryWrite(message); +#endif } public void Terminate() @@ -42,7 +58,11 @@ public void Terminate() Type = SnafflerMessageType.Fatal, Message = "Terminate was called" }); +#if NETFRAMEWORK + _collection.CompleteAdding(); +#else _channel.Writer.TryComplete(); +#endif } public void Trace(string message) @@ -126,7 +146,11 @@ public void Finish() DateTime = DateTime.Now, Type = SnafflerMessageType.Finish }); +#if NETFRAMEWORK + _collection.CompleteAdding(); +#else _channel.Writer.TryComplete(); +#endif } } -} \ No newline at end of file +} diff --git a/SnaffCore/Config/CsTomlStub.cs b/SnaffCore/Config/CsTomlStub.cs new file mode 100644 index 00000000..55cd7a40 --- /dev/null +++ b/SnaffCore/Config/CsTomlStub.cs @@ -0,0 +1,11 @@ +#if NETFRAMEWORK +namespace CsToml +{ + [System.AttributeUsage(System.AttributeTargets.Class | System.AttributeTargets.Struct)] + public sealed class TomlSerializedObjectAttribute : System.Attribute { } + + [System.AttributeUsage(System.AttributeTargets.Property | System.AttributeTargets.Field)] + public sealed class TomlValueOnSerializedAttribute : System.Attribute { } +} +#endif + diff --git a/SnaffCore/SnaffCore.csproj b/SnaffCore/SnaffCore.csproj index 1ff530a6..1c51aedc 100644 --- a/SnaffCore/SnaffCore.csproj +++ b/SnaffCore/SnaffCore.csproj @@ -1,13 +1,21 @@ - net9.0 + net9.0;net451 false + latest + + true - + + + + + + diff --git a/SnaffCore/UltraSnaffCore.csproj b/SnaffCore/UltraSnaffCore.csproj index 1fac9e66..f8e74c8d 100644 --- a/SnaffCore/UltraSnaffCore.csproj +++ b/SnaffCore/UltraSnaffCore.csproj @@ -1,11 +1,14 @@ - net9.0 + net9.0;net451 false ULTRASNAFFLER + latest + + true - + @@ -13,4 +16,11 @@ + + + + + + + diff --git a/Snaffler/Config.cs b/Snaffler/Config.cs index 39c38bcf..46589551 100644 --- a/Snaffler/Config.cs +++ b/Snaffler/Config.cs @@ -1,5 +1,4 @@ using CommandLineParser.Arguments; -using CsToml; using SnaffCore.Concurrency; using SnaffCore.Config; using System; @@ -138,7 +137,7 @@ private static async Task ParseImplAsync(string[] args) if ((args.Contains("--help") || args.Contains("/?") || args.Contains("help") || args.Contains("-h") || args.Length == 0)) { parser.ShowUsage(); - return null; + return null; } @@ -197,7 +196,7 @@ private static async Task ParseImplAsync(string[] args) if (compExclusionArg.Parsed) { List compExclusions = new List(); - string[] fileLines = await File.ReadAllLinesAsync(compExclusionArg.Value); + string[] fileLines = await FileCompat.ReadAllLinesAsync(compExclusionArg.Value); foreach (string line in fileLines) { if (isIP(line)) @@ -237,7 +236,7 @@ private static async Task ParseImplAsync(string[] args) List compTargets = new List(); if (compTargetArg.Value.Contains(Path.DirectorySeparatorChar)) { - string[] targetLines = await File.ReadAllLinesAsync(compTargetArg.Value); + string[] targetLines = await FileCompat.ReadAllLinesAsync(compTargetArg.Value); compTargets.AddRange(targetLines.Select(line => line.Trim())); } else if (compTargetArg.Value.Contains(",")) @@ -363,8 +362,7 @@ private static async Task ParseImplAsync(string[] args) { if (configFileArg.Value.Equals("generate")) { - using var result = CsTomlSerializer.Serialize(parsedConfig); - File.WriteAllBytes(".\\default.toml", result.ByteSpan.ToArray()); + await TomlCompat.WriteFileAsync(parsedConfig, ".\\default.toml"); Console.WriteLine("Wrote config values to .\\default.toml"); parsedConfig.LogToConsole = true; Mq.Degub("Enabled logging to stdout."); @@ -373,7 +371,7 @@ private static async Task ParseImplAsync(string[] args) else { string configFile = configFileArg.Value; - parsedConfig = CsTomlSerializer.Deserialize(await File.ReadAllBytesAsync(configFile)); + parsedConfig = await TomlCompat.ReadFileAsync(configFile); Mq.Info("Read config file from " + configFile); } } @@ -407,7 +405,7 @@ private static async Task ParseImplAsync(string[] args) string bulktoml = sb.ToString(); // deserialise the toml to an actual ruleset - RuleSet ruleSet = CsTomlSerializer.Deserialize(System.Text.Encoding.UTF8.GetBytes(bulktoml)); + RuleSet ruleSet = TomlCompat.ReadString(bulktoml); // stick the rules in our config! parsedConfig.ClassifierRules = ruleSet.ClassifierRules; @@ -418,12 +416,12 @@ private static async Task ParseImplAsync(string[] args) StringBuilder sb = new StringBuilder(); foreach (string tomlfile in tomlfiles) { - string tomlstring = await File.ReadAllTextAsync(tomlfile); + string tomlstring = await FileCompat.ReadAllTextAsync(tomlfile); sb.AppendLine(tomlstring); } string bulktoml = sb.ToString(); // deserialise the toml to an actual ruleset - RuleSet ruleSet = CsTomlSerializer.Deserialize(System.Text.Encoding.UTF8.GetBytes(bulktoml)); + RuleSet ruleSet = TomlCompat.ReadString(bulktoml); // stick the rules in our config! parsedConfig.ClassifierRules = ruleSet.ClassifierRules; diff --git a/Snaffler/FileCompat.cs b/Snaffler/FileCompat.cs new file mode 100644 index 00000000..0fd32fed --- /dev/null +++ b/Snaffler/FileCompat.cs @@ -0,0 +1,45 @@ +using System.IO; +using System.Threading.Tasks; + +namespace Snaffler +{ + internal static class FileCompat + { + public static Task ReadAllLinesAsync(string path) + { +#if NETFRAMEWORK + return Task.FromResult(File.ReadAllLines(path)); +#else + return File.ReadAllLinesAsync(path); +#endif + } + + public static Task ReadAllTextAsync(string path) + { +#if NETFRAMEWORK + return Task.FromResult(File.ReadAllText(path)); +#else + return File.ReadAllTextAsync(path); +#endif + } + + public static Task ReadAllBytesAsync(string path) + { +#if NETFRAMEWORK + return Task.FromResult(File.ReadAllBytes(path)); +#else + return File.ReadAllBytesAsync(path); +#endif + } + + public static Task WriteAllBytesAsync(string path, byte[] bytes) + { +#if NETFRAMEWORK + File.WriteAllBytes(path, bytes); + return Task.FromResult(0); +#else + return File.WriteAllBytesAsync(path, bytes); +#endif + } + } +} diff --git a/Snaffler/FodyWeavers.xml b/Snaffler/FodyWeavers.xml new file mode 100644 index 00000000..7bb3a417 --- /dev/null +++ b/Snaffler/FodyWeavers.xml @@ -0,0 +1,4 @@ + + + + diff --git a/Snaffler/IsExternalInit.cs b/Snaffler/IsExternalInit.cs new file mode 100644 index 00000000..341d210e --- /dev/null +++ b/Snaffler/IsExternalInit.cs @@ -0,0 +1,6 @@ +#if NETFRAMEWORK +namespace System.Runtime.CompilerServices +{ + public static class IsExternalInit { } +} +#endif diff --git a/Snaffler/Logger.cs b/Snaffler/Logger.cs index 9c3795c2..98d0f232 100644 --- a/Snaffler/Logger.cs +++ b/Snaffler/Logger.cs @@ -1,10 +1,15 @@ using System; using System.IO; using System.Collections.Generic; +#if NETFRAMEWORK +using System.Collections.Concurrent; +using Newtonsoft.Json; +#else using System.Threading.Channels; -using System.Threading.Tasks; using System.Text.Json; using System.Text.Json.Serialization; +#endif +using System.Threading.Tasks; using SnaffCore.Config; #nullable enable @@ -29,12 +34,17 @@ public static class Logger private static LogLevel _minLevel; private static LogType _logType; private static char _separator; +#if NETFRAMEWORK + private static BlockingCollection? _channel; + private static Task? _consumerTask; +#else private static Channel? _channel; private static Task? _consumerTask; private static readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; +#endif private readonly record struct LogEntry(string Prefix, string Tag, ConsoleColor TagColor, string Message, string FileMessage); @@ -52,13 +62,24 @@ public static void Configure(bool logToConsole, bool logToFile, string? filePath AutoFlush = true }; } +#if NETFRAMEWORK + _channel = new BlockingCollection(); + _consumerTask = Task.Run(ProcessQueueAsync); +#else _channel = Channel.CreateUnbounded(); _consumerTask = Task.Run(ProcessQueueAsync); +#endif } +#if NETFRAMEWORK + public static void Close() + { + _channel?.CompleteAdding(); +#else public static void Close() { _channel?.Writer.TryComplete(); +#endif _consumerTask?.Wait(); lock (_lock) { @@ -183,10 +204,17 @@ private static void WriteFile(string text) } } +#if NETFRAMEWORK + private static async Task ProcessQueueAsync() + { + if (_channel == null) return; + foreach (var entry in _channel.GetConsumingEnumerable()) +#else private static async Task ProcessQueueAsync() { if (_channel == null) return; await foreach (var entry in _channel.Reader.ReadAllAsync()) +#endif { if (_consoleEnabled) { @@ -199,6 +227,19 @@ private static async Task ProcessQueueAsync() } } +#if NETFRAMEWORK + private static string BuildJson(LogLevel level, DateTime time, string message, object? data) + { + var obj = new + { + time = time.ToUniversalTime().ToString("O"), + level = level.ToString(), + message, + eventProperties = data + }; + return JsonConvert.SerializeObject(obj); + } +#else private static string BuildJson(LogLevel level, DateTime time, string message, object? data) { var obj = new @@ -210,6 +251,7 @@ private static string BuildJson(LogLevel level, DateTime time, string message, o }; return JsonSerializer.Serialize(obj, _jsonOptions); } +#endif private static void Log(LogLevel level, DateTime time, string prefix, string tag, ConsoleColor tagColor, string message, object? data = null) { @@ -218,11 +260,19 @@ private static void Log(LogLevel level, DateTime time, string prefix, string tag string plain = string.Concat(prefix, tag, _separator, message); string output = _logType == LogType.JSON ? BuildJson(level, time, plain, data) : plain; var entry = new LogEntry(prefix, tag, tagColor, message, output); +#if NETFRAMEWORK + if (_channel != null) + { + _channel.Add(entry); + } + else +#else if (_channel != null) { _channel.Writer.TryWrite(entry); } else +#endif { if (_consoleEnabled) { diff --git a/Snaffler/Properties/PublishProfiles/net451.pubxml b/Snaffler/Properties/PublishProfiles/net451.pubxml new file mode 100644 index 00000000..03500749 --- /dev/null +++ b/Snaffler/Properties/PublishProfiles/net451.pubxml @@ -0,0 +1,11 @@ + + + net451 + win-x64 + Release + false + + + + + diff --git a/Snaffler/Properties/PublishProfiles/net9.pubxml b/Snaffler/Properties/PublishProfiles/net9.pubxml new file mode 100644 index 00000000..dd5e0220 --- /dev/null +++ b/Snaffler/Properties/PublishProfiles/net9.pubxml @@ -0,0 +1,9 @@ + + + net9.0 + win-x64 + true + true + Release + + diff --git a/Snaffler/SnaffleRunner.cs b/Snaffler/SnaffleRunner.cs index 43c4a447..94f11583 100644 --- a/Snaffler/SnaffleRunner.cs +++ b/Snaffler/SnaffleRunner.cs @@ -108,8 +108,13 @@ public async Task RunAsync(string[] args) private void DumpQueue() { +#if NETFRAMEWORK + BlockingMq Mq = BlockingMq.GetMq(); + while (Mq.Collection.TryTake(out SnafflerMessage message)) +#else BlockingMq Mq = BlockingMq.GetMq(); while (Mq.Reader.TryRead(out SnafflerMessage message)) +#endif { // emergency dump of queue contents to console Console.WriteLine(message.Message); @@ -123,7 +128,11 @@ private void DumpQueue() private async Task HandleOutputAsync() { BlockingMq Mq = BlockingMq.GetMq(); +#if NETFRAMEWORK + foreach (SnafflerMessage message in Mq.Collection.GetConsumingEnumerable()) +#else await foreach (SnafflerMessage message in Mq.Reader.ReadAllAsync()) +#endif { if (Options.LogType == LogType.Plain) { @@ -445,7 +454,7 @@ private async Task FixJSONOutputAsync() //Prepare the normalised file using StreamWriter file = new StreamWriter(Options.LogFilePath); //Read in the original log file to an array - string[] lines = await System.IO.File.ReadAllLinesAsync(Options.LogFilePath + ".tmp"); + string[] lines = await FileCompat.ReadAllLinesAsync(Options.LogFilePath + ".tmp"); //Write the surrounding template that we need. await file.WriteAsync("{\"entries\": [\n"); //Write all the lines into the new file but add a comma after all but the last so it becomes valid JSON. diff --git a/Snaffler/Snaffler.csproj b/Snaffler/Snaffler.csproj index 73ac66c7..2d5ee6a2 100644 --- a/Snaffler/Snaffler.csproj +++ b/Snaffler/Snaffler.csproj @@ -1,11 +1,18 @@ Exe - net9.0 + net9.0;net451 false Snaffler Snaffler + latest + + true + true + + + true @@ -16,8 +23,17 @@ - + + + + + + + + + + diff --git a/Snaffler/TomlCompat.cs b/Snaffler/TomlCompat.cs new file mode 100644 index 00000000..2577384e --- /dev/null +++ b/Snaffler/TomlCompat.cs @@ -0,0 +1,53 @@ +using System; +using System.Threading.Tasks; +using SnaffCore.Config; +#if NETFRAMEWORK +using Nett; +#else +using CsToml; +#endif + +namespace Snaffler +{ + internal static class TomlCompat + { +#if NETFRAMEWORK + private static readonly TomlSettings Settings = TomlSettings.Create(cfg => cfg + .ConfigureType(tc => tc.WithConversionFor(conv => conv + .FromToml(s => (LogLevel)Enum.Parse(typeof(LogLevel), s.Value, ignoreCase: true)) + .ToToml(e => e.ToString())))); + + public static Task WriteFileAsync(Options options, string path) + { + Toml.WriteFile(options, path, Settings); + return Task.FromResult(0); + } + + public static Task ReadFileAsync(string path) + { + return Task.FromResult(Toml.ReadFile(path, Settings)); + } + + public static RuleSet ReadString(string content) + { + return Toml.ReadString(content, Settings); + } +#else + public static async Task WriteFileAsync(Options options, string path) + { + using var result = CsTomlSerializer.Serialize(options); + await FileCompat.WriteAllBytesAsync(path, result.ByteSpan.ToArray()); + } + + public static async Task ReadFileAsync(string path) + { + return CsTomlSerializer.Deserialize(await FileCompat.ReadAllBytesAsync(path)); + } + + public static RuleSet ReadString(string content) + { + return CsTomlSerializer.Deserialize(System.Text.Encoding.UTF8.GetBytes(content)); + } +#endif + } +} diff --git a/publish.bat b/publish.bat new file mode 100644 index 00000000..f8ec9a40 --- /dev/null +++ b/publish.bat @@ -0,0 +1,9 @@ +@echo off +setlocal +set DOTNET_FLAGS=-c Release + +dotnet publish Snaffler\Snaffler.csproj %DOTNET_FLAGS% -f net451 -p:PublishProfile=net451 + +dotnet publish Snaffler\Snaffler.csproj %DOTNET_FLAGS% -f net9.0 -p:PublishProfile=net9 +endlocal + From 24f285a801d181d9881d7e9064868a7501b40e7d Mon Sep 17 00:00:00 2001 From: coj337 Date: Sun, 15 Jun 2025 12:53:38 +1000 Subject: [PATCH 10/10] Simplify framework conditional compilations --- Snaffler/Logger.cs | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/Snaffler/Logger.cs b/Snaffler/Logger.cs index 98d0f232..fa4d83b2 100644 --- a/Snaffler/Logger.cs +++ b/Snaffler/Logger.cs @@ -204,15 +204,12 @@ private static void WriteFile(string text) } } -#if NETFRAMEWORK private static async Task ProcessQueueAsync() { if (_channel == null) return; +#if NETFRAMEWORK foreach (var entry in _channel.GetConsumingEnumerable()) #else - private static async Task ProcessQueueAsync() - { - if (_channel == null) return; await foreach (var entry in _channel.Reader.ReadAllAsync()) #endif { @@ -227,7 +224,7 @@ private static async Task ProcessQueueAsync() } } -#if NETFRAMEWORK + private static string BuildJson(LogLevel level, DateTime time, string message, object? data) { var obj = new @@ -237,21 +234,12 @@ private static string BuildJson(LogLevel level, DateTime time, string message, o message, eventProperties = data }; +#if NETFRAMEWORK return JsonConvert.SerializeObject(obj); - } #else - private static string BuildJson(LogLevel level, DateTime time, string message, object? data) - { - var obj = new - { - time = time.ToUniversalTime().ToString("O"), - level = level.ToString(), - message, - eventProperties = data - }; return JsonSerializer.Serialize(obj, _jsonOptions); - } #endif + } private static void Log(LogLevel level, DateTime time, string prefix, string tag, ConsoleColor tagColor, string message, object? data = null) {