From 13775638a8a1f016fd90213194bf62f550342d02 Mon Sep 17 00:00:00 2001 From: Electrostats Date: Fri, 24 Nov 2023 06:58:11 -0500 Subject: [PATCH 1/9] Convert nullable bool to bool --- CustomTracks/CustomTrack.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CustomTracks/CustomTrack.cs b/CustomTracks/CustomTrack.cs index dcca86f..b33e74a 100644 --- a/CustomTracks/CustomTrack.cs +++ b/CustomTracks/CustomTrack.cs @@ -38,7 +38,7 @@ public CustomTrack(string folderPath, CustomTrackData data, TrackLoader loader) public SavedLevel LoadChart() { - return _loader?.ShouldReloadChart() == true ? _loader.ReloadTrack(this) : _data.ToSavedLevel(); + return (bool)_loader?.ShouldReloadChart() ? _loader.ReloadTrack(this) : _data.ToSavedLevel(); } public LoadedTromboneTrack LoadTrack() From c1ee8ac787766b8436bd4cb97b398573f67644bd Mon Sep 17 00:00:00 2001 From: Electrostats Date: Fri, 24 Nov 2023 10:44:15 -0500 Subject: [PATCH 2/9] Multi threaded chart loading + other optimizations -Iterating through custom songs folder only -Removed redundant customLevel null check -Add tracks to a List then return that list as enumerable -Removed serializer instance as it end up taking more memory from testing anyway Testing showed it improved loading time for ~1.2k charts from 3.5s down to 1.9s on average --- CustomTracks/TrackLoader.cs | 51 +++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/CustomTracks/TrackLoader.cs b/CustomTracks/TrackLoader.cs index cdb7c01..d7002b6 100644 --- a/CustomTracks/TrackLoader.cs +++ b/CustomTracks/TrackLoader.cs @@ -1,73 +1,70 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Runtime.Serialization; +using System.Threading.Tasks; using BaboonAPI.Hooks.Tracks; using Newtonsoft.Json; using TrombLoader.Helpers; namespace TrombLoader.CustomTracks; -public class TrackLoader: TrackRegistrationEvent.Listener +public class TrackLoader : TrackRegistrationEvent.Listener { - private JsonSerializer _serializer = new(); - public IEnumerable OnRegisterTracks() { CreateMissingDirectories(); - var songs = Directory.GetFiles(Globals.GetCustomSongsPath(), "song.tmb", SearchOption.AllDirectories) - .Concat(Directory.GetFiles(BepInEx.Paths.PluginPath, "song.tmb", SearchOption.AllDirectories)) - .Select(i => Path.GetDirectoryName(i)); + List tracks = new List(); + + //Non recursive method is faster but requires stricter custom songs folder structure + //var songDirectories = Directory.GetDirectories(Globals.GetCustomSongsPath()); + //Only recursively check inside of the custom songs folder + var songDirectories = Directory.GetFiles(Globals.GetCustomSongsPath(), "song.tmb", SearchOption.AllDirectories); var seen = new HashSet(); - foreach (var songFolder in songs) - { - var chartPath = Path.Combine(songFolder, Globals.defaultChartName); - var chartName = Path.GetFileName(songFolder.TrimEnd('/')); - if (!File.Exists(chartPath)) continue; - using var stream = File.OpenText(chartPath); - using var reader = new JsonTextReader(stream); + //Using more thread isn't faster, tested on 12 logical processor, best result using 3 + Parallel.For(0, songDirectories.Length, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount / 4 }, i => + { + if (!File.Exists(songDirectories[i])) return; CustomTrackData customLevel; try { - _serializer.Context = new StreamingContext(StreamingContextStates.File, chartName); - customLevel = _serializer.Deserialize(reader); + var chartName = Path.GetDirectoryName(songDirectories[i]).TrimEnd('/'); + customLevel = JsonConvert.DeserializeObject(File.ReadAllText(songDirectories[i]), new JsonSerializerSettings() + { + Context = new StreamingContext(StreamingContextStates.File, chartName) + }); } catch (Exception exc) { - Plugin.LogWarning($"Unable to deserialize JSON of custom chart: {chartPath}"); + Plugin.LogWarning($"Unable to deserialize JSON of custom chart: {songDirectories[i]}"); Plugin.LogWarning(exc.Message); - continue; + return; } - if (customLevel == null) continue; - if (seen.Add(customLevel.trackRef)) { Plugin.LogDebug($"Found custom chart: {customLevel.trackRef}"); - yield return new CustomTrack(songFolder, customLevel, this); + tracks.Add(new CustomTrack(songDirectories[i], customLevel, this)); } else { Plugin.LogWarning( - $"Skipping folder {chartPath} as its trackref '{customLevel.trackRef}' was already loaded!"); + $"Skipping folder {songDirectories[i]} as its trackref '{customLevel.trackRef}' was already loaded!"); } - } + }); + return tracks; } public SavedLevel ReloadTrack(CustomTrack existing) { var chartPath = Path.Combine(existing.folderPath, Globals.defaultChartName); - using var stream = File.OpenText(chartPath); - using var reader = new JsonTextReader(stream); - - var track = _serializer.Deserialize(reader); + var track = JsonConvert.DeserializeObject(File.ReadAllText(chartPath)); return track?.ToSavedLevel(); } From 6ee2706b790a4b6835b0f1746cae3073a52b5f2d Mon Sep 17 00:00:00 2001 From: Electrostats Date: Fri, 24 Nov 2023 10:52:55 -0500 Subject: [PATCH 3/9] Remove another redundant check --- CustomTracks/TrackLoader.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/CustomTracks/TrackLoader.cs b/CustomTracks/TrackLoader.cs index d7002b6..f452219 100644 --- a/CustomTracks/TrackLoader.cs +++ b/CustomTracks/TrackLoader.cs @@ -27,8 +27,6 @@ public IEnumerable OnRegisterTracks() //Using more thread isn't faster, tested on 12 logical processor, best result using 3 Parallel.For(0, songDirectories.Length, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount / 4 }, i => { - if (!File.Exists(songDirectories[i])) return; - CustomTrackData customLevel; try { From 183e10e1444d93fb6d09cfa2a4e4c38f473423fd Mon Sep 17 00:00:00 2001 From: Electrostats Date: Fri, 24 Nov 2023 10:58:50 -0500 Subject: [PATCH 4/9] Fixed passing the wrong directory path to CustomTrack constructor + variable renaming --- CustomTracks/TrackLoader.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/CustomTracks/TrackLoader.cs b/CustomTracks/TrackLoader.cs index f452219..99baf5e 100644 --- a/CustomTracks/TrackLoader.cs +++ b/CustomTracks/TrackLoader.cs @@ -21,24 +21,26 @@ public IEnumerable OnRegisterTracks() //var songDirectories = Directory.GetDirectories(Globals.GetCustomSongsPath()); //Only recursively check inside of the custom songs folder - var songDirectories = Directory.GetFiles(Globals.GetCustomSongsPath(), "song.tmb", SearchOption.AllDirectories); + var tmbDirectories = Directory.GetFiles(Globals.GetCustomSongsPath(), "song.tmb", SearchOption.AllDirectories); var seen = new HashSet(); //Using more thread isn't faster, tested on 12 logical processor, best result using 3 - Parallel.For(0, songDirectories.Length, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount / 4 }, i => + Parallel.For(0, tmbDirectories.Length, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount / 4 }, i => { CustomTrackData customLevel; + string dirName; try { - var chartName = Path.GetDirectoryName(songDirectories[i]).TrimEnd('/'); - customLevel = JsonConvert.DeserializeObject(File.ReadAllText(songDirectories[i]), new JsonSerializerSettings() + dirName = Path.GetDirectoryName(tmbDirectories[i]); + var chartName = dirName.TrimEnd('/'); + customLevel = JsonConvert.DeserializeObject(File.ReadAllText(tmbDirectories[i]), new JsonSerializerSettings() { Context = new StreamingContext(StreamingContextStates.File, chartName) }); } catch (Exception exc) { - Plugin.LogWarning($"Unable to deserialize JSON of custom chart: {songDirectories[i]}"); + Plugin.LogWarning($"Unable to deserialize JSON of custom chart: {tmbDirectories[i]}"); Plugin.LogWarning(exc.Message); return; } @@ -47,12 +49,12 @@ public IEnumerable OnRegisterTracks() { Plugin.LogDebug($"Found custom chart: {customLevel.trackRef}"); - tracks.Add(new CustomTrack(songDirectories[i], customLevel, this)); + tracks.Add(new CustomTrack(dirName, customLevel, this)); } else { Plugin.LogWarning( - $"Skipping folder {songDirectories[i]} as its trackref '{customLevel.trackRef}' was already loaded!"); + $"Skipping folder {dirName} as its trackref '{customLevel.trackRef}' was already loaded!"); } }); return tracks; From bc8c541eed4079667da8f7ff3aae0e0b46c64987 Mon Sep 17 00:00:00 2001 From: Electrostats Date: Fri, 24 Nov 2023 11:55:43 -0500 Subject: [PATCH 5/9] Added elapsed time to load charts + limited threads to 2 --- CustomTracks/TrackLoader.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CustomTracks/TrackLoader.cs b/CustomTracks/TrackLoader.cs index 99baf5e..e893eaa 100644 --- a/CustomTracks/TrackLoader.cs +++ b/CustomTracks/TrackLoader.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Runtime.Serialization; using System.Threading.Tasks; @@ -13,6 +14,7 @@ public class TrackLoader : TrackRegistrationEvent.Listener { public IEnumerable OnRegisterTracks() { + Stopwatch sw = Stopwatch.StartNew(); CreateMissingDirectories(); List tracks = new List(); @@ -24,8 +26,7 @@ public IEnumerable OnRegisterTracks() var tmbDirectories = Directory.GetFiles(Globals.GetCustomSongsPath(), "song.tmb", SearchOption.AllDirectories); var seen = new HashSet(); - //Using more thread isn't faster, tested on 12 logical processor, best result using 3 - Parallel.For(0, tmbDirectories.Length, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount / 4 }, i => + Parallel.For(0, tmbDirectories.Length, new ParallelOptions() { MaxDegreeOfParallelism = 2 }, i => { CustomTrackData customLevel; string dirName; @@ -57,6 +58,8 @@ public IEnumerable OnRegisterTracks() $"Skipping folder {dirName} as its trackref '{customLevel.trackRef}' was already loaded!"); } }); + sw.Stop(); + Plugin.LogInfo($"{tracks.Count} charts were loaded in {sw.Elapsed.TotalMilliseconds:0.00}ms"); return tracks; } From 622525230a533bcc5799c50a0fb3461fb601bee6 Mon Sep 17 00:00:00 2001 From: Electrostats Date: Fri, 24 Nov 2023 13:03:16 -0500 Subject: [PATCH 6/9] Added comment --- CustomTracks/TrackLoader.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/CustomTracks/TrackLoader.cs b/CustomTracks/TrackLoader.cs index e893eaa..c6022e2 100644 --- a/CustomTracks/TrackLoader.cs +++ b/CustomTracks/TrackLoader.cs @@ -26,6 +26,7 @@ public IEnumerable OnRegisterTracks() var tmbDirectories = Directory.GetFiles(Globals.GetCustomSongsPath(), "song.tmb", SearchOption.AllDirectories); var seen = new HashSet(); + //For some reasons more thread ends up being much slower even than single threading, but using a low thread count makes it much faster Parallel.For(0, tmbDirectories.Length, new ParallelOptions() { MaxDegreeOfParallelism = 2 }, i => { CustomTrackData customLevel; From 04f9efc30d3ea48093435e124123d6d11ab0b231 Mon Sep 17 00:00:00 2001 From: Electrostats Date: Fri, 24 Nov 2023 13:06:37 -0500 Subject: [PATCH 7/9] micro refactoring --- CustomTracks/Backgrounds/VideoBackground.cs | 8 ++------ CustomTracks/ChartCompatibility.cs | 4 ++-- CustomTracks/CustomTrack.cs | 5 +++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/CustomTracks/Backgrounds/VideoBackground.cs b/CustomTracks/Backgrounds/VideoBackground.cs index 1c5a64b..b9d9778 100644 --- a/CustomTracks/Backgrounds/VideoBackground.cs +++ b/CustomTracks/Backgrounds/VideoBackground.cs @@ -14,7 +14,7 @@ public VideoBackground(string videoPath) { _videoPath = videoPath; } - + public override void SetUpBackground(BGController controller, GameObject bg) { DisableParts(bg); @@ -56,10 +56,6 @@ public override void OnResume(PauseContext ctx) public static IEnumerable PlayVideoDelayed(VideoPlayer videoPlayer) { yield return new WaitForSeconds(2.4f); - - if (videoPlayer != null) - { - videoPlayer.Play(); - } + videoPlayer?.Play(); } } \ No newline at end of file diff --git a/CustomTracks/ChartCompatibility.cs b/CustomTracks/ChartCompatibility.cs index dd93958..3e3f89e 100644 --- a/CustomTracks/ChartCompatibility.cs +++ b/CustomTracks/ChartCompatibility.cs @@ -36,7 +36,7 @@ public override int ReadJson(JsonReader reader, Type objectType, int existingVal if (reader.TokenType is JsonToken.Float) { var chartName = (string) serializer.Context.Context; - _logger.LogWarning($"Chart '{chartName}' has invalid type {reader.TokenType.ToString()} on field {_fieldName} (expected an integer)"); + _logger.LogWarning($"Chart '{chartName}' has invalid type {reader.TokenType} on field {_fieldName} (expected an integer)"); } return (int) Convert.ToDouble(reader.Value); @@ -44,7 +44,7 @@ public override int ReadJson(JsonReader reader, Type objectType, int existingVal if (reader.TokenType != JsonToken.Integer) { - throw new JsonException($"Expected number, got {reader.TokenType.ToString()}"); + throw new JsonException($"Expected number, got {reader.TokenType}"); } return Convert.ToInt32(reader.Value); diff --git a/CustomTracks/CustomTrack.cs b/CustomTracks/CustomTrack.cs index b33e74a..dcb28a1 100644 --- a/CustomTracks/CustomTrack.cs +++ b/CustomTracks/CustomTrack.cs @@ -65,9 +65,10 @@ public Coroutines.YieldTask> LoadClip() private AbstractBackground LoadBackground() { - if (File.Exists(Path.Combine(folderPath, "bg.trombackground"))) + var possibleBackgroundPath = Path.Combine(folderPath, "bg.trombackground"); + if (File.Exists(possibleBackgroundPath)) { - var bundle = AssetBundle.LoadFromFile(Path.Combine(folderPath, "bg.trombackground")); + var bundle = AssetBundle.LoadFromFile(possibleBackgroundPath); return new CustomBackground(bundle, folderPath); } From 43a2a783af2876fca9ec84897286344e937c4096 Mon Sep 17 00:00:00 2001 From: Electrostats Date: Sun, 26 Nov 2023 23:20:46 -0500 Subject: [PATCH 8/9] Prevent very rare occasional boot crash --- CustomTracks/TrackLoader.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CustomTracks/TrackLoader.cs b/CustomTracks/TrackLoader.cs index c6022e2..c560548 100644 --- a/CustomTracks/TrackLoader.cs +++ b/CustomTracks/TrackLoader.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Runtime.Serialization; using System.Threading.Tasks; using BaboonAPI.Hooks.Tracks; @@ -38,7 +39,7 @@ public IEnumerable OnRegisterTracks() customLevel = JsonConvert.DeserializeObject(File.ReadAllText(tmbDirectories[i]), new JsonSerializerSettings() { Context = new StreamingContext(StreamingContextStates.File, chartName) - }); + }) ?? throw new Exception("Deserializer returned unexpected null value."); } catch (Exception exc) { @@ -61,7 +62,7 @@ public IEnumerable OnRegisterTracks() }); sw.Stop(); Plugin.LogInfo($"{tracks.Count} charts were loaded in {sw.Elapsed.TotalMilliseconds:0.00}ms"); - return tracks; + return tracks.Where(x => x != null); } public SavedLevel ReloadTrack(CustomTrack existing) From 8702ce059c612f05a5af7a3cbff860088279b44c Mon Sep 17 00:00:00 2001 From: Electrostats Date: Sun, 31 Dec 2023 23:59:42 -0500 Subject: [PATCH 9/9] Removed parallelisation --- CustomTracks/TrackLoader.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CustomTracks/TrackLoader.cs b/CustomTracks/TrackLoader.cs index c560548..c926957 100644 --- a/CustomTracks/TrackLoader.cs +++ b/CustomTracks/TrackLoader.cs @@ -4,7 +4,6 @@ using System.IO; using System.Linq; using System.Runtime.Serialization; -using System.Threading.Tasks; using BaboonAPI.Hooks.Tracks; using Newtonsoft.Json; using TrombLoader.Helpers; @@ -28,7 +27,7 @@ public IEnumerable OnRegisterTracks() var seen = new HashSet(); //For some reasons more thread ends up being much slower even than single threading, but using a low thread count makes it much faster - Parallel.For(0, tmbDirectories.Length, new ParallelOptions() { MaxDegreeOfParallelism = 2 }, i => + for(int i = 0; i < tmbDirectories.Length; i++) { CustomTrackData customLevel; string dirName; @@ -45,7 +44,7 @@ public IEnumerable OnRegisterTracks() { Plugin.LogWarning($"Unable to deserialize JSON of custom chart: {tmbDirectories[i]}"); Plugin.LogWarning(exc.Message); - return; + continue; } if (seen.Add(customLevel.trackRef)) @@ -59,7 +58,7 @@ public IEnumerable OnRegisterTracks() Plugin.LogWarning( $"Skipping folder {dirName} as its trackref '{customLevel.trackRef}' was already loaded!"); } - }); + }; sw.Stop(); Plugin.LogInfo($"{tracks.Count} charts were loaded in {sw.Elapsed.TotalMilliseconds:0.00}ms"); return tracks.Where(x => x != null);