From 5561265f334b7e42e351ab3c1d14c6eed2e5f5ac Mon Sep 17 00:00:00 2001 From: Slendy Date: Thu, 29 Jun 2023 00:36:18 -0500 Subject: [PATCH 1/8] Initial work for score migration --- .../Resources/ResourcesController.cs | 4 +- .../Controllers/Slots/ScoreController.cs | 113 ++++++------- .../Pages/Partials/LeaderboardPartial.cshtml | 15 +- .../Maintenance/MaintenanceHelper.cs | 17 +- .../CleanupBrokenVersusScoresMigration.cs | 24 --- .../CleanupDuplicateScoresMigration.cs | 51 ------ .../MigrationTasks/CleanupSanitizedStrings.cs | 6 +- .../CleanupSlotVersionMismatchMigration.cs | 6 +- .../SwitchScoreToUserIdMigration.cs | 149 ++++++++++++++++++ ...0628043618_AddUserIdAndTimestampToScore.cs | 41 +++++ .../Migrations/DatabaseModelSnapshot.cs | 6 + ProjectLighthouse/StartupTasks.cs | 92 +++++++++-- .../Types/Entities/Level/ScoreEntity.cs | 27 ++-- .../Maintenance/CompletedMigrationEntity.cs | 2 +- .../{IMigrationTask.cs => MigrationTask.cs} | 19 ++- .../Types/Serialization/GameScore.cs | 18 ++- 16 files changed, 387 insertions(+), 203 deletions(-) delete mode 100644 ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupBrokenVersusScoresMigration.cs delete mode 100644 ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupDuplicateScoresMigration.cs create mode 100644 ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs create mode 100644 ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs rename ProjectLighthouse/Types/Maintenance/{IMigrationTask.cs => MigrationTask.cs} (60%) diff --git a/ProjectLighthouse.Servers.GameServer/Controllers/Resources/ResourcesController.cs b/ProjectLighthouse.Servers.GameServer/Controllers/Resources/ResourcesController.cs index 7973c9a85..9c541c686 100644 --- a/ProjectLighthouse.Servers.GameServer/Controllers/Resources/ResourcesController.cs +++ b/ProjectLighthouse.Servers.GameServer/Controllers/Resources/ResourcesController.cs @@ -42,11 +42,11 @@ public IActionResult GetResource(string hash) string fullPath = Path.GetFullPath(path); // Prevent directory traversal attacks - if (!fullPath.StartsWith(FileHelper.FullResourcePath)) return this.BadRequest(); + if (!fullPath.StartsWith(FileHelper.FullResourcePath)) return this.StatusCode(400); if (FileHelper.ResourceExists(hash)) return this.File(IOFile.OpenRead(path), "application/octet-stream"); - return this.NotFound(); + return this.StatusCode(404); } // TODO: check if this is a valid hash diff --git a/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs b/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs index cfb986b11..dd1ffdb40 100644 --- a/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs +++ b/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs @@ -30,26 +30,15 @@ public ScoreController(DatabaseContext database) this.database = database; } - private string[] getFriendUsernames(int userId, string username) + private static int[] GetFriendIds(int userId) { UserFriendData? store = UserFriendStore.GetUserFriendData(userId); - if (store == null) return new[] { username, }; - - List friendNames = new() + List? friendIds = store?.FriendIds; + friendIds ??= new List { - username, + userId, }; - - // ReSharper disable once ForeachCanBeConvertedToQueryUsingAnotherGetEnumerator - foreach (int friendId in store.FriendIds) - { - string? friendUsername = this.database.Users.Where(u => u.UserId == friendId) - .Select(u => u.Username) - .FirstOrDefault(); - if (friendUsername != null) friendNames.Add(friendUsername); - } - - return friendNames.ToArray(); + return friendIds.Append(userId).Distinct().ToArray(); } [HttpPost("scoreboard/{slotType}/{id:int}")] @@ -144,33 +133,33 @@ public async Task SubmitScore(string slotType, int id, int childI await this.database.SaveChangesAsync(); - string playerIdCollection = string.Join(',', score.PlayerIds); - ScoreEntity? existingScore = await this.database.Scores.Where(s => s.SlotId == slot.SlotId) .Where(s => s.ChildSlotId == 0 || s.ChildSlotId == childId) - .Where(s => s.PlayerIdCollection == playerIdCollection) + .Where(s => s.UserId == token.UserId) .Where(s => s.Type == score.Type) .FirstOrDefaultAsync(); if (existingScore != null) { existingScore.Points = Math.Max(existingScore.Points, score.Points); + existingScore.Timestamp = TimeHelper.TimestampMillis; } else { ScoreEntity playerScore = new() { - PlayerIdCollection = playerIdCollection, + UserId = token.UserId, Type = score.Type, Points = score.Points, SlotId = slotId, ChildSlotId = childId, + Timestamp = TimeHelper.TimestampMillis, }; this.database.Scores.Add(playerScore); } await this.database.SaveChangesAsync(); - return this.Ok(this.getScores(new LeaderboardOptions + return this.Ok(await this.GetScores(new LeaderboardOptions { RootName = "scoreboardSegment", PageSize = 5, @@ -178,7 +167,7 @@ public async Task SubmitScore(string slotType, int id, int childI SlotId = slotId, ChildSlotId = childId, ScoreType = score.Type, - TargetUsername = username, + TargetUser = token.UserId, TargetPlayerIds = null, })); } @@ -189,8 +178,6 @@ public async Task Lbp1Leaderboards(string slotType, int id) { GameTokenEntity token = this.GetToken(); - string username = await this.database.UsernameFromGameToken(token); - if (slotType == "developer") id = await SlotHelper.GetPlaceholderSlotId(this.database, id, SlotType.Developer); LeaderboardOptions options = new() @@ -199,7 +186,7 @@ public async Task Lbp1Leaderboards(string slotType, int id) PageStart = 1, ScoreType = -1, SlotId = id, - TargetUsername = username, + TargetUser = token.UserId, RootName = "scoreboardSegment", }; if (!HttpMethods.IsPost(this.Request.Method)) @@ -208,7 +195,7 @@ public async Task Lbp1Leaderboards(string slotType, int id) for (int i = 1; i <= 4; i++) { options.ScoreType = i; - ScoreboardResponse response = this.getScores(options); + ScoreboardResponse response = await this.GetScores(options); scoreboardResponses.Add(new PlayerScoreboardResponse(response.Scores, i)); } return this.Ok(new MultiScoreboardResponse(scoreboardResponses)); @@ -217,9 +204,9 @@ public async Task Lbp1Leaderboards(string slotType, int id) GameScore? score = await this.DeserializeBody(); if (score == null) return this.BadRequest(); options.ScoreType = score.Type; - options.TargetPlayerIds = this.getFriendUsernames(token.UserId, username); + options.TargetPlayerIds = GetFriendIds(token.UserId); - return this.Ok(this.getScores(options)); + return this.Ok(await this.GetScores(options)); } [HttpGet("friendscores/{slotType}/{slotId:int}/{type:int}")] @@ -230,15 +217,13 @@ public async Task FriendScores(string slotType, int slotId, int? if (pageSize <= 0) return this.BadRequest(); - string username = await this.database.UsernameFromGameToken(token); - if (SlotHelper.IsTypeInvalid(slotType)) return this.BadRequest(); if (slotType == "developer") slotId = await SlotHelper.GetPlaceholderSlotId(this.database, slotId, SlotType.Developer); - string[] friendIds = this.getFriendUsernames(token.UserId, username); + int[] friendIds = GetFriendIds(token.UserId); - return this.Ok(this.getScores(new LeaderboardOptions + return this.Ok(await this.GetScores(new LeaderboardOptions { RootName = "scores", PageSize = pageSize, @@ -246,27 +231,24 @@ public async Task FriendScores(string slotType, int slotId, int? SlotId = slotId, ChildSlotId = childId, ScoreType = type, - TargetUsername = username, + TargetUser = token.UserId, TargetPlayerIds = friendIds, })); } [HttpGet("topscores/{slotType}/{slotId:int}/{type:int}")] [HttpGet("topscores/{slotType}/{slotId:int}/{childId:int}/{type:int}")] - [SuppressMessage("ReSharper", "PossibleMultipleEnumeration")] public async Task TopScores(string slotType, int slotId, int? childId, int type, [FromQuery] int pageStart = -1, [FromQuery] int pageSize = 5) { GameTokenEntity token = this.GetToken(); if (pageSize <= 0) return this.BadRequest(); - string username = await this.database.UsernameFromGameToken(token); - if (SlotHelper.IsTypeInvalid(slotType)) return this.BadRequest(); if (slotType == "developer") slotId = await SlotHelper.GetPlaceholderSlotId(this.database, slotId, SlotType.Developer); - return this.Ok(this.getScores(new LeaderboardOptions + return this.Ok(await this.GetScores(new LeaderboardOptions { RootName = "scores", PageSize = pageSize, @@ -274,52 +256,55 @@ public async Task TopScores(string slotType, int slotId, int? chi SlotId = slotId, ChildSlotId = childId, ScoreType = type, - TargetUsername = username, + TargetUser = token.UserId, TargetPlayerIds = null, })); } private class LeaderboardOptions { - public int SlotId { get; set; } + public int SlotId { get; init; } public int ScoreType { get; set; } - public string TargetUsername { get; set; } = ""; - public int PageStart { get; set; } = -1; - public int PageSize { get; set; } = 5; - public string RootName { get; set; } = "scores"; - public string[]? TargetPlayerIds; + public int TargetUser { get; init; } + public int PageStart { get; init; } = -1; + public int PageSize { get; init; } = 5; + public string RootName { get; init; } = "scores"; + public int[]? TargetPlayerIds; public int? ChildSlotId; } - private ScoreboardResponse getScores(LeaderboardOptions options) + private async Task GetScores(LeaderboardOptions options) { - // This is hella ugly but it technically assigns the proper rank to a score - // var needed for Anonymous type returned from SELECT - var rankedScores = this.database.Scores.Where(s => s.SlotId == options.SlotId) + IQueryable scoreQuery = this.database.Scores.Where(s => s.SlotId == options.SlotId) .Where(s => options.ScoreType == -1 || s.Type == options.ScoreType) .Where(s => s.ChildSlotId == 0 || s.ChildSlotId == options.ChildSlotId) - .AsEnumerable() - .Where(s => options.TargetPlayerIds == null || - options.TargetPlayerIds.Any(id => s.PlayerIdCollection.Split(",").Contains(id))) - .OrderByDescending(s => s.Points) - .ThenBy(s => s.ScoreId) - .ToList() - .Select((s, rank) => new + .Where(s => options.TargetPlayerIds == null || options.TargetPlayerIds.Contains(s.UserId)); + + // First find if you have a score on a level to find scores around it + var myScore = await scoreQuery.Where(s => s.UserId == options.TargetUser) + .Select(s => new { Score = s, - Rank = rank + 1, - }) - .ToList(); + Rank = scoreQuery.Count(s2 => s2.Points > s.Points)+1, + }).FirstOrDefaultAsync(); + int skipAmt = options.PageStart != -1 || myScore == null ? options.PageStart - 1 : myScore.Rank - 3; - // Find your score, since even if you aren't in the top list your score is pinned - var myScore = rankedScores.Where(rs => rs.Score.PlayerIdCollection.Split(",").Contains(options.TargetUsername)).MaxBy(rs => rs.Score.Points); + var rankedScores = scoreQuery.OrderByDescending(s => s.Points) + .ThenBy(s => s.ScoreId) + .Skip(Math.Max(0, skipAmt)) + .Take(Math.Min(options.PageSize, 30)) + .Select(s => new + { + Score = s, + Rank = scoreQuery.Count(s2 => s2.Points > s.Points)+1, + }) + .ToList(); - // Paginated viewing: if not requesting pageStart, get results around user - var pagedScores = rankedScores.Skip(options.PageStart != -1 || myScore == null ? options.PageStart - 1 : myScore.Rank - 3).Take(Math.Min(options.PageSize, 30)); + int totalScores = scoreQuery.Count(); - List gameScores = pagedScores.ToSerializableList(ps => GameScore.CreateFromEntity(ps.Score, ps.Rank)); + List gameScores = rankedScores.ToSerializableList(ps => GameScore.CreateFromEntity(ps.Score, ps.Rank)); - return new ScoreboardResponse(options.RootName, gameScores, rankedScores.Count, myScore?.Score.Points ?? 0, myScore?.Rank ?? 0); + return new ScoreboardResponse(options.RootName, gameScores, totalScores, myScore?.Score.Points ?? 0, myScore?.Rank ?? 0); } } \ No newline at end of file diff --git a/ProjectLighthouse.Servers.Website/Pages/Partials/LeaderboardPartial.cshtml b/ProjectLighthouse.Servers.Website/Pages/Partials/LeaderboardPartial.cshtml index 797fac6d2..7081ca61c 100644 --- a/ProjectLighthouse.Servers.Website/Pages/Partials/LeaderboardPartial.cshtml +++ b/ProjectLighthouse.Servers.Website/Pages/Partials/LeaderboardPartial.cshtml @@ -24,7 +24,6 @@ @for(int i = 0; i < Model.Scores.Count; i++) { ScoreEntity score = Model.Scores[i]; - string[] playerIds = score.PlayerIds; DatabaseContext database = Model.Database;
@@ -39,9 +38,11 @@
- @for (int j = 0; j < playerIds.Length; j++) - { - UserEntity? user = await database.Users.FirstOrDefaultAsync(u => u.Username == playerIds[j]); + @* @for (int j = 0; j < userIds.Length; j++) *@ + @* { *@ + @{ + UserEntity? user = await database.Users.FindAsync(score.UserId); + }
@@ -49,13 +50,9 @@ { @await user.ToLink(Html, ViewData, language, timeZone) } - else - { -

@playerIds[j]

- }
- } + // }
diff --git a/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs b/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs index dd17d8e31..27659d7b1 100644 --- a/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs +++ b/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs @@ -19,15 +19,15 @@ public static class MaintenanceHelper { static MaintenanceHelper() { - Commands = getListOfInterfaceObjects(); - MaintenanceJobs = getListOfInterfaceObjects(); - MigrationTasks = getListOfInterfaceObjects(); - RepeatingTasks = getListOfInterfaceObjects(); + Commands = GetListOfInterfaceObjects(); + MaintenanceJobs = GetListOfInterfaceObjects(); + MigrationTasks = GetListOfInterfaceObjects(); + RepeatingTasks = GetListOfInterfaceObjects(); } public static List Commands { get; } public static List MaintenanceJobs { get; } - public static List MigrationTasks { get; } + public static List MigrationTasks { get; } public static List RepeatingTasks { get; } public static async Task> RunCommand(IServiceProvider provider, string[] args) @@ -80,9 +80,8 @@ public static async Task RunMaintenanceJob(string jobName) await job.Run(); } - public static async Task RunMigration(DatabaseContext database, IMigrationTask migrationTask) + public static async Task RunMigration(DatabaseContext database, MigrationTask migrationTask) { - // Migrations should never be run twice. Debug.Assert(!await database.CompletedMigrations.Has(m => m.MigrationName == migrationTask.GetType().Name)); @@ -121,11 +120,11 @@ public static async Task RunMigration(DatabaseContext database, IMigrationTask m await database.SaveChangesAsync(); } - private static List getListOfInterfaceObjects() where T : class + private static List GetListOfInterfaceObjects() where T : class { return Assembly.GetExecutingAssembly() .GetTypes() - .Where(t => t.GetInterfaces().Contains(typeof(T)) && t.GetConstructor(Type.EmptyTypes) != null) + .Where(t => (t.IsSubclassOf(typeof(T)) || t.GetInterfaces().Contains(typeof(T))) && t.GetConstructor(Type.EmptyTypes) != null) .Select(t => Activator.CreateInstance(t) as T) .ToList()!; } diff --git a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupBrokenVersusScoresMigration.cs b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupBrokenVersusScoresMigration.cs deleted file mode 100644 index 67a9ad14b..000000000 --- a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupBrokenVersusScoresMigration.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System.Threading.Tasks; -using LBPUnion.ProjectLighthouse.Database; -using LBPUnion.ProjectLighthouse.Types.Entities.Level; -using LBPUnion.ProjectLighthouse.Types.Maintenance; - -namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks; - -public class FixBrokenVersusScores : IMigrationTask -{ - public string Name() => "Cleanup versus scores"; - - async Task IMigrationTask.Run(DatabaseContext database) - { - foreach (ScoreEntity score in database.Scores) - { - if (!score.PlayerIdCollection.Contains(':')) continue; - - score.PlayerIdCollection = score.PlayerIdCollection.Replace(':', ','); - } - - await database.SaveChangesAsync(); - return true; - } -} \ No newline at end of file diff --git a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupDuplicateScoresMigration.cs b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupDuplicateScoresMigration.cs deleted file mode 100644 index 1a7e25e93..000000000 --- a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupDuplicateScoresMigration.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using LBPUnion.ProjectLighthouse.Database; -using LBPUnion.ProjectLighthouse.Logging; -using LBPUnion.ProjectLighthouse.Types.Entities.Level; -using LBPUnion.ProjectLighthouse.Types.Logging; -using LBPUnion.ProjectLighthouse.Types.Maintenance; - -namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks; - -public class CleanupDuplicateScoresMigration : IMigrationTask -{ - public string Name() => "Cleanup duplicate scores"; - - public async Task Run(DatabaseContext database) - { - List duplicateScoreIds = new(); - // The original score should have the lowest score id - foreach (ScoreEntity score in database.Scores.OrderBy(s => s.ScoreId) - .ToList() - .Where(score => !duplicateScoreIds.Contains(score.ScoreId))) - { - foreach (ScoreEntity other in database.Scores.Where(s => - s.Points == score.Points && - s.Type == score.Type && - s.SlotId == score.SlotId && - s.ScoreId != score.ScoreId && - s.ChildSlotId == score.ChildSlotId && - s.ScoreId > score.ScoreId)) - { - if (score.PlayerIds.Length != other.PlayerIds.Length) - continue; - - HashSet hashSet = new(score.PlayerIds); - - if (!other.PlayerIds.All(hashSet.Contains)) continue; - - Logger.Info($"Removing score with id {other.ScoreId}, slotId={other.SlotId} main='{score.PlayerIdCollection}', duplicate={other.PlayerIdCollection}", LogArea.Score); - database.Scores.Remove(other); - duplicateScoreIds.Add(other.ScoreId); - } - } - - Logger.Info($"Removed a total of {duplicateScoreIds.Count} duplicate scores", LogArea.Score); - await database.SaveChangesAsync(); - - return true; - } - -} \ No newline at end of file diff --git a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupSanitizedStrings.cs b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupSanitizedStrings.cs index c039b05de..47a38a60a 100644 --- a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupSanitizedStrings.cs +++ b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupSanitizedStrings.cs @@ -8,11 +8,11 @@ namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks; -public class CleanupSanitizedStrings : IMigrationTask +public class CleanupSanitizedStrings : MigrationTask { - public string Name() => "Cleanup Sanitized strings"; + public override string Name() => "Cleanup Sanitized strings"; - async Task IMigrationTask.Run(DatabaseContext database) + public override async Task Run(DatabaseContext database) { List objsToBeSanitized = new(); diff --git a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupSlotVersionMismatchMigration.cs b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupSlotVersionMismatchMigration.cs index df07e7e54..136bd0709 100644 --- a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupSlotVersionMismatchMigration.cs +++ b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupSlotVersionMismatchMigration.cs @@ -8,11 +8,11 @@ namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks; -public class CleanupSlotVersionMismatchMigration : IMigrationTask +public class CleanupSlotVersionMismatchMigration : MigrationTask { - public string Name() => "Cleanup slot versions"; + public override string Name() => "Cleanup slot versions"; - async Task IMigrationTask.Run(DatabaseContext database) + public override async Task Run(DatabaseContext database) { foreach (SlotEntity slot in database.Slots) { diff --git a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs new file mode 100644 index 000000000..191bd47a8 --- /dev/null +++ b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; +using System.Linq; +using System.Threading.Tasks; +using LBPUnion.ProjectLighthouse.Configuration; +using LBPUnion.ProjectLighthouse.Database; +using LBPUnion.ProjectLighthouse.Helpers; +using LBPUnion.ProjectLighthouse.Types.Entities.Level; +using LBPUnion.ProjectLighthouse.Types.Entities.Profile; +using LBPUnion.ProjectLighthouse.Types.Maintenance; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks; + +public class SwitchScoreToUserIdMigration : MigrationTask +{ + #region DB entity replication stuff + + [Table("Scores")] + private class OldScoreEntity + { + [Key] + public int ScoreId { get; set; } + + public int SlotId { get; set; } + + [ForeignKey(nameof(SlotId))] + public SlotEntity Slot { get; set; } + + public int ChildSlotId { get; set; } + + public int Type { get; set; } + + public string PlayerIdCollection { get; set; } + + [NotMapped] + public string[] PlayerIds + { + get => this.PlayerIdCollection.Split(","); + set => this.PlayerIdCollection = string.Join(',', value); + } + + public int UserId { get; set; } + + public int Points { get; set; } + + public long Timestamp { get; set; } + } + + private sealed class CustomDbContext : DbContext + { + public CustomDbContext(DbContextOptions options) : base(options) + { } + + public DbSet Scores { get; set; } + public DbSet Slots { get; set; } + public DbSet Users { get; set; } + } + + #endregion + + public override string Name() => "20230628023610_AddUserIdAndTimestampToScore"; + + public override MigrationHook HookType() => MigrationHook.After; + + private List GetAllSlots(DatabaseContext database) + { + return null; + } + + public override async Task Run(DatabaseContext db) + { + DbContextOptionsBuilder builder = new(); + builder.UseMySql(ServerConfiguration.Instance.DbConnectionString, + MySqlServerVersion.LatestSupportedServerVersion); + + int[] scoreTypes = + { + 1, 2, 3, 4, 7, + }; + CustomDbContext database = new(builder.Options); + List newScores = new(); + // Get all slots with at least 1 score + foreach (SlotEntity slot in await database.Slots + .Where(s => database.Scores.Count(score => s.SlotId == score.SlotId) > 0) + .ToListAsync()) + { + foreach (int type in scoreTypes) + { + newScores.AddRange(await FixScores(database, slot, type)); + } + } + + return true; + } + + private static Dictionary CreateHighestScores(CustomDbContext database, List scores) + { + Dictionary maxPointsByPlayer = new(); + foreach (OldScoreEntity score in scores) + { + string[] players = score.PlayerIds; + foreach (string player in players) + { + if (!database.Users.Any(u => u.Username == player)) continue; + + _ = maxPointsByPlayer.TryGetValue(player, out int highestScore); + highestScore = Math.Max(highestScore, score.Points); + maxPointsByPlayer[player] = highestScore; + } + } + + return maxPointsByPlayer; + } + + private static async Task> FixScores(CustomDbContext database, SlotEntity slot, int scoreType) + { + //TODO create a map of all players with scores submitted, then find their highest score for this type and create a new score + List newScores = new(); + + // Loop over all scores for this level grouped by ChildSlotId (to account for adventure levels) + foreach (IGrouping group in database.Scores.Where(s => s.SlotId == slot.SlotId) + .Where(s => s.Type == scoreType) + .GroupBy(s => s.ChildSlotId)) + { + Dictionary highestScores = CreateHighestScores(database, group.ToList()); + foreach (KeyValuePair kvp in highestScores) + { + int userId = await database.Users.Where(u => u.Username == kvp.Key).Select(u => u.UserId).FirstAsync(); + ScoreEntity scoreEntity = new() + { + UserId = userId, + SlotId = slot.SlotId, + ChildSlotId = group.Key, + Points = kvp.Value, + Timestamp = TimeHelper.TimestampMillis, + Type = scoreType, + }; + newScores.Add(scoreEntity); + } + } + + return newScores; + } + +} \ No newline at end of file diff --git a/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs b/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs new file mode 100644 index 000000000..15c38f923 --- /dev/null +++ b/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs @@ -0,0 +1,41 @@ +using LBPUnion.ProjectLighthouse.Database; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ProjectLighthouse.Migrations +{ + [DbContext(typeof(DatabaseContext))] + [Migration("20230628043618_AddUserIdAndTimestampToScore")] + public partial class AddUserIdAndTimestampToScore : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "Timestamp", + table: "Scores", + type: "bigint", + nullable: false, + defaultValue: 0L); + + migrationBuilder.AddColumn( + name: "UserId", + table: "Scores", + type: "int", + nullable: false, + defaultValue: 0); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "Timestamp", + table: "Scores"); + + migrationBuilder.DropColumn( + name: "UserId", + table: "Scores"); + } + } +} diff --git a/ProjectLighthouse/ProjectLighthouse/Migrations/DatabaseModelSnapshot.cs b/ProjectLighthouse/ProjectLighthouse/Migrations/DatabaseModelSnapshot.cs index 2e5991fa1..55e6e3e1a 100644 --- a/ProjectLighthouse/ProjectLighthouse/Migrations/DatabaseModelSnapshot.cs +++ b/ProjectLighthouse/ProjectLighthouse/Migrations/DatabaseModelSnapshot.cs @@ -330,9 +330,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("SlotId") .HasColumnType("int"); + b.Property("Timestamp") + .HasColumnType("bigint"); + b.Property("Type") .HasColumnType("int"); + b.Property("UserId") + .HasColumnType("int"); + b.HasKey("ScoreId"); b.HasIndex("SlotId"); diff --git a/ProjectLighthouse/StartupTasks.cs b/ProjectLighthouse/StartupTasks.cs index 2fdfc76a9..2a49a5151 100644 --- a/ProjectLighthouse/StartupTasks.cs +++ b/ProjectLighthouse/StartupTasks.cs @@ -1,11 +1,16 @@ using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; using System.Diagnostics; using System.Linq; using System.Reflection; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using System.Transactions; using LBPUnion.ProjectLighthouse.Administration.Maintenance; +using LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks; using LBPUnion.ProjectLighthouse.Configuration; using LBPUnion.ProjectLighthouse.Database; using LBPUnion.ProjectLighthouse.Extensions; @@ -14,19 +19,52 @@ using LBPUnion.ProjectLighthouse.Logging; using LBPUnion.ProjectLighthouse.Logging.Loggers; using LBPUnion.ProjectLighthouse.StorableLists; +using LBPUnion.ProjectLighthouse.Types.Entities.Level; using LBPUnion.ProjectLighthouse.Types.Entities.Maintenance; using LBPUnion.ProjectLighthouse.Types.Entities.Profile; using LBPUnion.ProjectLighthouse.Types.Logging; using LBPUnion.ProjectLighthouse.Types.Maintenance; -using LBPUnion.ProjectLighthouse.Types.Misc; using LBPUnion.ProjectLighthouse.Types.Users; using Medallion.Threading.MySql; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using Pomelo.EntityFrameworkCore.MySql.Infrastructure; +using ServerType = LBPUnion.ProjectLighthouse.Types.Misc.ServerType; namespace LBPUnion.ProjectLighthouse; public static class StartupTasks { + + private class OldScoreEntity + { + public int ScoreId { get; set; } + + public int SlotId { get; set; } + + public int ChildSlotId { get; set; } + + public int Type { get; set; } + + public string PlayerIdCollection { get; set; } + + [NotMapped] + public string[] PlayerIds + { + get => this.PlayerIdCollection.Split(","); + set => this.PlayerIdCollection = string.Join(',', value); + } + + public int UserId { get; set; } + + public int Points { get; set; } + + public long Timestamp { get; set; } + } public static async Task Run(ServerType serverType) { // Log startup time @@ -161,24 +199,50 @@ private static async Task MigrateDatabase(DatabaseContext database) Logger.Success($"Acquiring migration lock took {stopwatch.ElapsedMilliseconds}ms", LogArea.Database); stopwatch.Restart(); - await database.Database.MigrateAsync(); - stopwatch.Stop(); - Logger.Success($"Structure migration took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database); - - stopwatch.Restart(); + List pendingMigrations = (await database.Database.GetPendingMigrationsAsync()).ToList(); + IMigrator migrator = database.GetInfrastructure().GetService(); - List completedMigrations = database.CompletedMigrations.ToList(); - List migrationsToRun = MaintenanceHelper.MigrationTasks - .Where(migrationTask => !completedMigrations - .Select(m => m.MigrationName) - .Contains(migrationTask.GetType().Name) - ).ToList(); + async Task RunLighthouseMigrations(Func predicate) + { + List tasks = MaintenanceHelper.MigrationTasks + .Where(predicate) + .ToList(); + foreach (MigrationTask task in tasks) + { + await MaintenanceHelper.RunMigration(database, task); + } + } - foreach (IMigrationTask migrationTask in migrationsToRun) + Logger.Info($"There are {pendingMigrations.Count} pending migrations", LogArea.Database); + foreach (string migration in pendingMigrations) { - MaintenanceHelper.RunMigration(database, migrationTask).Wait(); + try + { + stopwatch.Restart(); + await using IDbContextTransaction transaction = await database.Database.BeginTransactionAsync(); + await RunLighthouseMigrations(m => m.Name() == migration && m.HookType() == MigrationHook.Before); + await migrator.MigrateAsync(migration); + await RunLighthouseMigrations(m => m.Name() == migration && m.HookType() == MigrationHook.After); + await transaction.CommitAsync(); + stopwatch.Stop(); + Logger.Success($"Running migration '{migration}' took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database); + + } + catch (Exception e) + { + await database.Database.RollbackTransactionAsync(); + Logger.Error($"Failed to run migration '{migration}'", LogArea.Database); + Logger.Error(e.ToDetailedException(), LogArea.Database); + Environment.Exit(-1); + } } + stopwatch.Restart(); + + List completedMigrations = database.CompletedMigrations.Select(m => m.MigrationName).ToList(); + + await RunLighthouseMigrations(m => m.HookType() == MigrationHook.None && !completedMigrations.Contains(m.Name())); + stopwatch.Stop(); totalStopwatch.Stop(); Logger.Success($"Extra migration tasks took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database); diff --git a/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs b/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs index ffc3fbb73..6c383f897 100644 --- a/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs +++ b/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs @@ -1,6 +1,6 @@ using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; -using System.Xml.Serialization; +using LBPUnion.ProjectLighthouse.Types.Entities.Profile; namespace LBPUnion.ProjectLighthouse.Types.Entities.Level; @@ -8,25 +8,24 @@ public class ScoreEntity { [Key] public int ScoreId { get; set; } - + public int SlotId { get; set; } - - [XmlIgnore] + [ForeignKey(nameof(SlotId))] public SlotEntity Slot { get; set; } - - [XmlIgnore] + public int ChildSlotId { get; set; } - + public int Type { get; set; } + + public int UserId { get; set; } + + // [ForeignKey(nameof(UserId))] + // public UserEntity User { get; set; } public string PlayerIdCollection { get; set; } - - [NotMapped] - public string[] PlayerIds { - get => this.PlayerIdCollection.Split(","); - set => this.PlayerIdCollection = string.Join(',', value); - } - + public int Points { get; set; } + + public long Timestamp { get; set; } } \ No newline at end of file diff --git a/ProjectLighthouse/Types/Entities/Maintenance/CompletedMigrationEntity.cs b/ProjectLighthouse/Types/Entities/Maintenance/CompletedMigrationEntity.cs index 55cda6855..245ab17eb 100644 --- a/ProjectLighthouse/Types/Entities/Maintenance/CompletedMigrationEntity.cs +++ b/ProjectLighthouse/Types/Entities/Maintenance/CompletedMigrationEntity.cs @@ -5,7 +5,7 @@ namespace LBPUnion.ProjectLighthouse.Types.Entities.Maintenance; /// -/// A record of the completion of a . +/// A record of the completion of a . /// public class CompletedMigrationEntity { diff --git a/ProjectLighthouse/Types/Maintenance/IMigrationTask.cs b/ProjectLighthouse/Types/Maintenance/MigrationTask.cs similarity index 60% rename from ProjectLighthouse/Types/Maintenance/IMigrationTask.cs rename to ProjectLighthouse/Types/Maintenance/MigrationTask.cs index 774a6cff1..326f98d55 100644 --- a/ProjectLighthouse/Types/Maintenance/IMigrationTask.cs +++ b/ProjectLighthouse/Types/Maintenance/MigrationTask.cs @@ -1,21 +1,28 @@ using System.Threading.Tasks; -using JetBrains.Annotations; using LBPUnion.ProjectLighthouse.Database; namespace LBPUnion.ProjectLighthouse.Types.Maintenance; -[UsedImplicitly(ImplicitUseTargetFlags.WithInheritors)] -public interface IMigrationTask +public enum MigrationHook +{ + Before, + After, + None, +} + +public abstract class MigrationTask { /// /// The user-friendly name of a migration. /// - public string Name(); - + public abstract string Name(); + + public virtual MigrationHook HookType() => MigrationHook.None; + /// /// Performs the migration. /// /// The Lighthouse database. /// True if successful, false if not. - internal Task Run(DatabaseContext database); + public abstract Task Run(DatabaseContext database); } \ No newline at end of file diff --git a/ProjectLighthouse/Types/Serialization/GameScore.cs b/ProjectLighthouse/Types/Serialization/GameScore.cs index 7addcf22c..dc33a6333 100644 --- a/ProjectLighthouse/Types/Serialization/GameScore.cs +++ b/ProjectLighthouse/Types/Serialization/GameScore.cs @@ -1,7 +1,10 @@ using System.ComponentModel; using System.Linq; +using System.Threading.Tasks; using System.Xml.Serialization; +using LBPUnion.ProjectLighthouse.Database; using LBPUnion.ProjectLighthouse.Types.Entities.Level; +using Microsoft.EntityFrameworkCore; namespace LBPUnion.ProjectLighthouse.Types.Serialization; @@ -9,9 +12,13 @@ namespace LBPUnion.ProjectLighthouse.Types.Serialization; [XmlType("playRecord")] public class GameScore : ILbpSerializable { + [XmlIgnore] + public int UserId { get; set; } + [XmlElement("type")] public int Type { get; set; } + [DefaultValue(null)] [XmlElement("playerIds")] public string[] PlayerIds; @@ -26,14 +33,19 @@ public class GameScore : ILbpSerializable [XmlElement("score")] public int Points { get; set; } + public async Task PrepareSerialization(DatabaseContext database) + { + this.MainPlayer = await database.Users.Where(u => u.UserId == this.UserId) + .Select(u => u.Username) + .FirstAsync(); + } + public static GameScore CreateFromEntity(ScoreEntity entity, int rank) => new() { - MainPlayer = entity.PlayerIds.ElementAtOrDefault(0) ?? "", - PlayerIds = entity.PlayerIds, + UserId = entity.UserId, Points = entity.Points, Type = entity.Type, Rank = rank, }; - } \ No newline at end of file From d01a31c4017181a55901379d9f3378eb1d6fa7d3 Mon Sep 17 00:00:00 2001 From: Slendy Date: Thu, 6 Jul 2023 00:35:36 -0500 Subject: [PATCH 2/8] Finish score migration --- .../Pages/Partials/LeaderboardPartial.cshtml | 3 - .../Maintenance/MaintenanceHelper.cs | 19 +- .../SwitchScoreToUserIdMigration.cs | 237 ++++++++++++------ ...pPlayerIdCollectionAndAddUserForeignKey.cs | 51 ++++ .../Migrations/DatabaseModelSnapshot.cs | 15 +- ProjectLighthouse/StartupTasks.cs | 59 ++--- .../Types/Entities/Level/ScoreEntity.cs | 6 +- .../Types/Maintenance/MigrationTask.cs | 1 - 8 files changed, 257 insertions(+), 134 deletions(-) create mode 100644 ProjectLighthouse/Migrations/20230706020914_DropPlayerIdCollectionAndAddUserForeignKey.cs diff --git a/ProjectLighthouse.Servers.Website/Pages/Partials/LeaderboardPartial.cshtml b/ProjectLighthouse.Servers.Website/Pages/Partials/LeaderboardPartial.cshtml index 7081ca61c..d2c865986 100644 --- a/ProjectLighthouse.Servers.Website/Pages/Partials/LeaderboardPartial.cshtml +++ b/ProjectLighthouse.Servers.Website/Pages/Partials/LeaderboardPartial.cshtml @@ -38,8 +38,6 @@
- @* @for (int j = 0; j < userIds.Length; j++) *@ - @* { *@ @{ UserEntity? user = await database.Users.FindAsync(score.UserId); } @@ -52,7 +50,6 @@ }
- // } diff --git a/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs b/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs index 27659d7b1..f87252a0f 100644 --- a/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs +++ b/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs @@ -80,15 +80,18 @@ public static async Task RunMaintenanceJob(string jobName) await job.Run(); } - public static async Task RunMigration(DatabaseContext database, MigrationTask migrationTask) + public static async Task RunMigration(DatabaseContext database, MigrationTask migrationTask) { // Migrations should never be run twice. - Debug.Assert(!await database.CompletedMigrations.Has(m => m.MigrationName == migrationTask.GetType().Name)); + Debug.Assert(!await database.CompletedMigrations.Has(m => m.MigrationName == migrationTask.GetType().Name), + $"Tried to run migration {migrationTask.GetType().Name} twice"); - Logger.Info($"Running migration task {migrationTask.Name()}", LogArea.Database); + Logger.Info($"Running LH migration task {migrationTask.Name()}", LogArea.Database); bool success; Exception? exception = null; + + Stopwatch stopwatch = Stopwatch.StartNew(); try { @@ -102,13 +105,14 @@ public static async Task RunMigration(DatabaseContext database, MigrationTask mi if (!success) { - Logger.Error($"Could not run migration {migrationTask.Name()}", LogArea.Database); + Logger.Error($"Could not run LH migration {migrationTask.Name()}", LogArea.Database); if (exception != null) Logger.Error(exception.ToDetailedException(), LogArea.Database); - return; + return false; } - - Logger.Success($"Successfully completed migration {migrationTask.Name()}", LogArea.Database); + stopwatch.Stop(); + + Logger.Success($"Successfully completed LH migration {migrationTask.Name()} in {stopwatch.ElapsedMilliseconds}", LogArea.Database); CompletedMigrationEntity completedMigration = new() { @@ -118,6 +122,7 @@ public static async Task RunMigration(DatabaseContext database, MigrationTask mi database.CompletedMigrations.Add(completedMigration); await database.SaveChangesAsync(); + return true; } private static List GetListOfInterfaceObjects() where T : class diff --git a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs index 191bd47a8..6fb0d2f7b 100644 --- a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs +++ b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs @@ -1,17 +1,20 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; +using System.Data; +using System.Data.Common; using System.Linq; +using System.Text; using System.Threading.Tasks; -using LBPUnion.ProjectLighthouse.Configuration; using LBPUnion.ProjectLighthouse.Database; using LBPUnion.ProjectLighthouse.Helpers; using LBPUnion.ProjectLighthouse.Types.Entities.Level; -using LBPUnion.ProjectLighthouse.Types.Entities.Profile; using LBPUnion.ProjectLighthouse.Types.Maintenance; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks; @@ -19,17 +22,12 @@ public class SwitchScoreToUserIdMigration : MigrationTask { #region DB entity replication stuff - [Table("Scores")] - private class OldScoreEntity + private class MigrationScore { - [Key] public int ScoreId { get; set; } public int SlotId { get; set; } - [ForeignKey(nameof(SlotId))] - public SlotEntity Slot { get; set; } - public int ChildSlotId { get; set; } public int Type { get; set; } @@ -37,75 +35,183 @@ private class OldScoreEntity public string PlayerIdCollection { get; set; } [NotMapped] - public string[] PlayerIds - { - get => this.PlayerIdCollection.Split(","); - set => this.PlayerIdCollection = string.Join(',', value); - } - - public int UserId { get; set; } + public IEnumerable PlayerIds => this.PlayerIdCollection.Split(","); public int Points { get; set; } + } - public long Timestamp { get; set; } + private class MigrationSlot + { + public int SlotId { get; set; } } - private sealed class CustomDbContext : DbContext + private class MigrationUser { - public CustomDbContext(DbContextOptions options) : base(options) - { } + public int UserId { get; set; } - public DbSet Scores { get; set; } - public DbSet Slots { get; set; } - public DbSet Users { get; set; } + public string Username { get; set; } } #endregion - public override string Name() => "20230628023610_AddUserIdAndTimestampToScore"; + public override string Name() => "20230706020914_DropPlayerIdCollectionAndAddUserForeignKey"; - public override MigrationHook HookType() => MigrationHook.After; + public override MigrationHook HookType() => MigrationHook.Before; - private List GetAllSlots(DatabaseContext database) + private static DbTransaction GetDbTransaction(IDbContextTransaction dbContextTransaction) { - return null; + if (dbContextTransaction is not IInfrastructure accessor) + { + throw new InvalidOperationException(RelationalStrings.RelationalNotInUse); + } + + return accessor.GetInfrastructure(); } - public override async Task Run(DatabaseContext db) + private static async Task> GetAllObjects(DbContext database, string commandText, Func returnFunc) { - DbContextOptionsBuilder builder = new(); - builder.UseMySql(ServerConfiguration.Instance.DbConnectionString, - MySqlServerVersion.LatestSupportedServerVersion); + DbConnection dbConnection = database.Database.GetDbConnection(); + + await using DbCommand cmd = dbConnection.CreateCommand(); + cmd.CommandText = commandText; + cmd.Transaction = GetDbTransaction(database.Database.CurrentTransaction); + + await using DbDataReader reader = await cmd.ExecuteReaderAsync(); + List items = new(); + + if (!reader.HasRows) return default; + while (await reader.ReadAsync()) + { + items.Add(returnFunc(reader)); + } + return items; + } + + private static async Task> GetAllScores(DbContext database) + { + return await GetAllObjects(database, + "select * from Scores", + reader => new MigrationScore + { + ScoreId = reader.GetInt32("ScoreId"), + SlotId = reader.GetInt32("SlotId"), + ChildSlotId = reader.GetInt32("ChildSlotId"), + Type = reader.GetInt32("Type"), + PlayerIdCollection = reader.GetString("PlayerIdCollection"), + Points = reader.GetInt32("Points"), + }); + } + + private static async Task> GetAllUsers(DbContext database) + { + return await GetAllObjects(database, + "select UserId, Username from Users", + reader => new MigrationUser() + { + UserId = reader.GetInt32("UserId"), + Username = reader.GetString("Username"), + }); + } + + private static async Task> GetAllSlots(DbContext database) + { + return await GetAllObjects(database, + "select SlotId from Slots", + reader => new MigrationSlot + { + SlotId = reader.GetInt32("SlotId"), + }); + } + + private static async Task ApplyFixedScores(DatabaseContext database, IReadOnlyList newScores) + { + // Re-order scores (The order doesn't make any difference but since we're already deleting everything we may as well) + newScores = newScores.OrderByDescending(s => s.SlotId) + .ThenByDescending(s => s.ChildSlotId) + .ThenByDescending(s => s.Type) + .ToList(); + + // Set IDs for new scores + for (int i = 1; i < newScores.Count; i++) + { + newScores[i].ScoreId = i; + } + // Delete all existing scores + await database.Scores.ExecuteDeleteAsync(); + + StringBuilder insertionScript = new(); + // This is significantly faster than using standard EntityFramework Add and Save albeit a little wacky + foreach (ScoreEntity score in newScores) + { + insertionScript.AppendLine($"""insert into Scores values('{score.ScoreId}', '{score.SlotId}', '{score.Type}', '', '{score.Points}', '{score.ChildSlotId}', '{score.Timestamp}', '{score.UserId}');"""); + } + + await database.Database.ExecuteSqlRawAsync(insertionScript.ToString()); + } + + public override async Task Run(DatabaseContext database) + { int[] scoreTypes = { 1, 2, 3, 4, 7, }; - CustomDbContext database = new(builder.Options); - List newScores = new(); + ConcurrentBag newScores = new(); // Get all slots with at least 1 score - foreach (SlotEntity slot in await database.Slots - .Where(s => database.Scores.Count(score => s.SlotId == score.SlotId) > 0) - .ToListAsync()) + List slots = await GetAllSlots(database); + List scores = await GetAllScores(database); + // Don't run migration if there are no scores + if (scores == null || scores.Count == 0) return true; + + List users = await GetAllUsers(database); + + ConcurrentQueue<(MigrationSlot slot, int type)> collection = new(); + foreach (MigrationSlot slot in slots.Where(s => scores.Any(score => s.SlotId == score.SlotId))) { foreach (int type in scoreTypes) { - newScores.AddRange(await FixScores(database, slot, type)); + collection.Enqueue((slot, type)); } } + ConcurrentBag taskList = new(); + for (int i = 0; i < Environment.ProcessorCount; i++) + { + Task task = Task.Run(() => + { + while (collection.TryDequeue(out (MigrationSlot slot, int type) item)) + { + List fixedScores = FixScores(users, + item.slot, + scores.Where(s => s.SlotId == item.slot.SlotId).Where(s => s.Type == item.type).ToList(), + item.type) + .ToList(); + fixedScores.AsParallel().ForAll(score => newScores.Add(score)); + } + }); + taskList.Add(task); + } + + await Task.WhenAll(taskList); + + await ApplyFixedScores(database, newScores.ToList()); + return true; } - private static Dictionary CreateHighestScores(CustomDbContext database, List scores) + /// + /// + /// + private static Dictionary CreateHighestScores(List scores, IReadOnlyCollection userCache) { - Dictionary maxPointsByPlayer = new(); - foreach (OldScoreEntity score in scores) + Dictionary maxPointsByPlayer = new(StringComparer.InvariantCultureIgnoreCase); + foreach (MigrationScore score in scores) { - string[] players = score.PlayerIds; + IEnumerable players = score.PlayerIds; foreach (string player in players) { - if (!database.Users.Any(u => u.Username == player)) continue; + // Remove non existent users to ensure foreign key constraint + if (userCache.All(u => u.Username != player)) continue; _ = maxPointsByPlayer.TryGetValue(player, out int highestScore); highestScore = Math.Max(highestScore, score.Points); @@ -116,34 +222,25 @@ private static Dictionary CreateHighestScores(CustomDbContext datab return maxPointsByPlayer; } - private static async Task> FixScores(CustomDbContext database, SlotEntity slot, int scoreType) + /// + /// This function groups slots by ChildSlotId to account for adventure scores and then for each user + /// finds their highest score on that level and adds a new Score + /// + private static IEnumerable FixScores(IReadOnlyCollection userCache, MigrationSlot slot, IEnumerable scores, int scoreType) { - //TODO create a map of all players with scores submitted, then find their highest score for this type and create a new score - List newScores = new(); - - // Loop over all scores for this level grouped by ChildSlotId (to account for adventure levels) - foreach (IGrouping group in database.Scores.Where(s => s.SlotId == slot.SlotId) - .Where(s => s.Type == scoreType) - .GroupBy(s => s.ChildSlotId)) - { - Dictionary highestScores = CreateHighestScores(database, group.ToList()); - foreach (KeyValuePair kvp in highestScores) + return ( + from slotGroup in scores.GroupBy(s => s.ChildSlotId) + let highestScores = CreateHighestScores(slotGroup.ToList(), userCache) + from kvp in highestScores + let userId = userCache.Where(u => u.Username == kvp.Key).Select(u => u.UserId).First() + select new ScoreEntity { - int userId = await database.Users.Where(u => u.Username == kvp.Key).Select(u => u.UserId).FirstAsync(); - ScoreEntity scoreEntity = new() - { - UserId = userId, - SlotId = slot.SlotId, - ChildSlotId = group.Key, - Points = kvp.Value, - Timestamp = TimeHelper.TimestampMillis, - Type = scoreType, - }; - newScores.Add(scoreEntity); - } - } - - return newScores; + UserId = userId, + SlotId = slot.SlotId, + ChildSlotId = slotGroup.Key, + Points = kvp.Value, + Timestamp = TimeHelper.TimestampMillis, + Type = scoreType, + }).ToList(); } - } \ No newline at end of file diff --git a/ProjectLighthouse/Migrations/20230706020914_DropPlayerIdCollectionAndAddUserForeignKey.cs b/ProjectLighthouse/Migrations/20230706020914_DropPlayerIdCollectionAndAddUserForeignKey.cs new file mode 100644 index 000000000..609741f9d --- /dev/null +++ b/ProjectLighthouse/Migrations/20230706020914_DropPlayerIdCollectionAndAddUserForeignKey.cs @@ -0,0 +1,51 @@ +using LBPUnion.ProjectLighthouse.Database; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ProjectLighthouse.Migrations +{ + [DbContext(typeof(DatabaseContext))] + [Migration("20230706020914_DropPlayerIdCollectionAndAddUserForeignKey")] + public partial class DropPlayerIdCollectionAndAddUserForeignKey : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "PlayerIdCollection", + table: "Scores"); + + migrationBuilder.CreateIndex( + name: "IX_Scores_UserId", + table: "Scores", + column: "UserId"); + + migrationBuilder.AddForeignKey( + name: "FK_Scores_Users_UserId", + table: "Scores", + column: "UserId", + principalTable: "Users", + principalColumn: "UserId", + onDelete: ReferentialAction.Cascade); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_Scores_Users_UserId", + table: "Scores"); + + migrationBuilder.DropIndex( + name: "IX_Scores_UserId", + table: "Scores"); + + migrationBuilder.AddColumn( + name: "PlayerIdCollection", + table: "Scores", + type: "longtext", + nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"); + } + } +} diff --git a/ProjectLighthouse/ProjectLighthouse/Migrations/DatabaseModelSnapshot.cs b/ProjectLighthouse/ProjectLighthouse/Migrations/DatabaseModelSnapshot.cs index 55e6e3e1a..6c0806d9a 100644 --- a/ProjectLighthouse/ProjectLighthouse/Migrations/DatabaseModelSnapshot.cs +++ b/ProjectLighthouse/ProjectLighthouse/Migrations/DatabaseModelSnapshot.cs @@ -16,7 +16,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "7.0.7") + .HasAnnotation("ProductVersion", "7.0.8") .HasAnnotation("Relational:MaxIdentifierLength", 64); modelBuilder.Entity("LBPUnion.ProjectLighthouse.Types.Entities.Interaction.HeartedLevelEntity", b => @@ -321,9 +321,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("ChildSlotId") .HasColumnType("int"); - b.Property("PlayerIdCollection") - .HasColumnType("longtext"); - b.Property("Points") .HasColumnType("int"); @@ -343,6 +340,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("SlotId"); + b.HasIndex("UserId"); + b.ToTable("Scores"); }); @@ -1242,7 +1241,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) .OnDelete(DeleteBehavior.Cascade) .IsRequired(); + b.HasOne("LBPUnion.ProjectLighthouse.Types.Entities.Profile.UserEntity", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + b.Navigation("Slot"); + + b.Navigation("User"); }); modelBuilder.Entity("LBPUnion.ProjectLighthouse.Types.Entities.Level.SlotEntity", b => diff --git a/ProjectLighthouse/StartupTasks.cs b/ProjectLighthouse/StartupTasks.cs index 2a49a5151..32a92941b 100644 --- a/ProjectLighthouse/StartupTasks.cs +++ b/ProjectLighthouse/StartupTasks.cs @@ -1,16 +1,11 @@ using System; using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; -using System.ComponentModel.DataAnnotations.Schema; using System.Diagnostics; using System.Linq; using System.Reflection; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using System.Transactions; using LBPUnion.ProjectLighthouse.Administration.Maintenance; -using LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks; using LBPUnion.ProjectLighthouse.Configuration; using LBPUnion.ProjectLighthouse.Database; using LBPUnion.ProjectLighthouse.Extensions; @@ -19,8 +14,6 @@ using LBPUnion.ProjectLighthouse.Logging; using LBPUnion.ProjectLighthouse.Logging.Loggers; using LBPUnion.ProjectLighthouse.StorableLists; -using LBPUnion.ProjectLighthouse.Types.Entities.Level; -using LBPUnion.ProjectLighthouse.Types.Entities.Maintenance; using LBPUnion.ProjectLighthouse.Types.Entities.Profile; using LBPUnion.ProjectLighthouse.Types.Logging; using LBPUnion.ProjectLighthouse.Types.Maintenance; @@ -28,43 +21,15 @@ using Medallion.Threading.MySql; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Metadata.Builders; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.DependencyInjection; -using Pomelo.EntityFrameworkCore.MySql.Infrastructure; using ServerType = LBPUnion.ProjectLighthouse.Types.Misc.ServerType; namespace LBPUnion.ProjectLighthouse; public static class StartupTasks { - - private class OldScoreEntity - { - public int ScoreId { get; set; } - - public int SlotId { get; set; } - - public int ChildSlotId { get; set; } - - public int Type { get; set; } - - public string PlayerIdCollection { get; set; } - - [NotMapped] - public string[] PlayerIds - { - get => this.PlayerIdCollection.Split(","); - set => this.PlayerIdCollection = string.Join(',', value); - } - - public int UserId { get; set; } - - public int Points { get; set; } - - public long Timestamp { get; set; } - } public static async Task Run(ServerType serverType) { // Log startup time @@ -200,39 +165,43 @@ private static async Task MigrateDatabase(DatabaseContext database) stopwatch.Restart(); List pendingMigrations = (await database.Database.GetPendingMigrationsAsync()).ToList(); - IMigrator migrator = database.GetInfrastructure().GetService(); + IMigrator migrator = database.GetInfrastructure().GetRequiredService(); - async Task RunLighthouseMigrations(Func predicate) + async Task RunLighthouseMigrations(Func predicate) { List tasks = MaintenanceHelper.MigrationTasks .Where(predicate) .ToList(); foreach (MigrationTask task in tasks) { - await MaintenanceHelper.RunMigration(database, task); + if (!await MaintenanceHelper.RunMigration(database, task)) return false; } + return true; } Logger.Info($"There are {pendingMigrations.Count} pending migrations", LogArea.Database); + foreach (string migration in pendingMigrations) { try { - stopwatch.Restart(); await using IDbContextTransaction transaction = await database.Database.BeginTransactionAsync(); - await RunLighthouseMigrations(m => m.Name() == migration && m.HookType() == MigrationHook.Before); + Logger.Debug($"Running migration '{migration}", LogArea.Database); + stopwatch.Restart(); + if (!await RunLighthouseMigrations(m => m.Name() == migration && m.HookType() == MigrationHook.Before)) + throw new Exception($"Failed to run pre migration hook for {migration}"); + await migrator.MigrateAsync(migration); - await RunLighthouseMigrations(m => m.Name() == migration && m.HookType() == MigrationHook.After); - await transaction.CommitAsync(); + stopwatch.Stop(); Logger.Success($"Running migration '{migration}' took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database); - } catch (Exception e) { - await database.Database.RollbackTransactionAsync(); Logger.Error($"Failed to run migration '{migration}'", LogArea.Database); Logger.Error(e.ToDetailedException(), LogArea.Database); + if (database.Database.CurrentTransaction != null) + await database.Database.RollbackTransactionAsync(); Environment.Exit(-1); } } @@ -241,7 +210,7 @@ async Task RunLighthouseMigrations(Func predicate) List completedMigrations = database.CompletedMigrations.Select(m => m.MigrationName).ToList(); - await RunLighthouseMigrations(m => m.HookType() == MigrationHook.None && !completedMigrations.Contains(m.Name())); + await RunLighthouseMigrations(m => !completedMigrations.Contains(m.GetType().Name) && m.HookType() == MigrationHook.None); stopwatch.Stop(); totalStopwatch.Stop(); diff --git a/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs b/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs index 6c383f897..2a3b82c75 100644 --- a/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs +++ b/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs @@ -20,11 +20,9 @@ public class ScoreEntity public int UserId { get; set; } - // [ForeignKey(nameof(UserId))] - // public UserEntity User { get; set; } + [ForeignKey(nameof(UserId))] + public UserEntity User { get; set; } - public string PlayerIdCollection { get; set; } - public int Points { get; set; } public long Timestamp { get; set; } diff --git a/ProjectLighthouse/Types/Maintenance/MigrationTask.cs b/ProjectLighthouse/Types/Maintenance/MigrationTask.cs index 326f98d55..a1c6899a6 100644 --- a/ProjectLighthouse/Types/Maintenance/MigrationTask.cs +++ b/ProjectLighthouse/Types/Maintenance/MigrationTask.cs @@ -6,7 +6,6 @@ namespace LBPUnion.ProjectLighthouse.Types.Maintenance; public enum MigrationHook { Before, - After, None, } From 541e2bdd7969337b7088d85241bd4dbe4915e4dd Mon Sep 17 00:00:00 2001 From: Slendy Date: Fri, 14 Jul 2023 17:10:54 -0500 Subject: [PATCH 3/8] Implement suggested changes from code review --- .../Resources/ResourcesController.cs | 4 +- .../SwitchScoreToUserIdMigration.cs | 50 ++++++------------- ProjectLighthouse/Helpers/MigrationHelper.cs | 44 ++++++++++++++++ .../Types/Entities/Level/ScoreEntity.cs | 12 ++--- 4 files changed, 66 insertions(+), 44 deletions(-) create mode 100644 ProjectLighthouse/Helpers/MigrationHelper.cs diff --git a/ProjectLighthouse.Servers.GameServer/Controllers/Resources/ResourcesController.cs b/ProjectLighthouse.Servers.GameServer/Controllers/Resources/ResourcesController.cs index 9c541c686..7973c9a85 100644 --- a/ProjectLighthouse.Servers.GameServer/Controllers/Resources/ResourcesController.cs +++ b/ProjectLighthouse.Servers.GameServer/Controllers/Resources/ResourcesController.cs @@ -42,11 +42,11 @@ public IActionResult GetResource(string hash) string fullPath = Path.GetFullPath(path); // Prevent directory traversal attacks - if (!fullPath.StartsWith(FileHelper.FullResourcePath)) return this.StatusCode(400); + if (!fullPath.StartsWith(FileHelper.FullResourcePath)) return this.BadRequest(); if (FileHelper.ResourceExists(hash)) return this.File(IOFile.OpenRead(path), "application/octet-stream"); - return this.StatusCode(404); + return this.NotFound(); } // TODO: check if this is a valid hash diff --git a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs index 6fb0d2f7b..2f846161d 100644 --- a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs +++ b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs @@ -58,39 +58,9 @@ private class MigrationUser public override MigrationHook HookType() => MigrationHook.Before; - private static DbTransaction GetDbTransaction(IDbContextTransaction dbContextTransaction) - { - if (dbContextTransaction is not IInfrastructure accessor) - { - throw new InvalidOperationException(RelationalStrings.RelationalNotInUse); - } - - return accessor.GetInfrastructure(); - } - - private static async Task> GetAllObjects(DbContext database, string commandText, Func returnFunc) - { - DbConnection dbConnection = database.Database.GetDbConnection(); - - await using DbCommand cmd = dbConnection.CreateCommand(); - cmd.CommandText = commandText; - cmd.Transaction = GetDbTransaction(database.Database.CurrentTransaction); - - await using DbDataReader reader = await cmd.ExecuteReaderAsync(); - List items = new(); - - if (!reader.HasRows) return default; - - while (await reader.ReadAsync()) - { - items.Add(returnFunc(reader)); - } - return items; - } - private static async Task> GetAllScores(DbContext database) { - return await GetAllObjects(database, + return await MigrationHelper.GetAllObjects(database, "select * from Scores", reader => new MigrationScore { @@ -105,9 +75,9 @@ private static async Task> GetAllScores(DbContext database) private static async Task> GetAllUsers(DbContext database) { - return await GetAllObjects(database, + return await MigrationHelper.GetAllObjects(database, "select UserId, Username from Users", - reader => new MigrationUser() + reader => new MigrationUser { UserId = reader.GetInt32("UserId"), Username = reader.GetString("Username"), @@ -116,7 +86,7 @@ private static async Task> GetAllUsers(DbContext database) private static async Task> GetAllSlots(DbContext database) { - return await GetAllObjects(database, + return await MigrationHelper.GetAllObjects(database, "select SlotId from Slots", reader => new MigrationSlot { @@ -124,6 +94,10 @@ private static async Task> GetAllSlots(DbContext database) }); } + /// + /// This function deletes all existing scores and inserts the new generated scores + /// All scores must be deleted because MySQL doesn't allow you to change primary keys + /// private static async Task ApplyFixedScores(DatabaseContext database, IReadOnlyList newScores) { // Re-order scores (The order doesn't make any difference but since we're already deleting everything we may as well) @@ -144,7 +118,10 @@ private static async Task ApplyFixedScores(DatabaseContext database, IReadOnlyLi // This is significantly faster than using standard EntityFramework Add and Save albeit a little wacky foreach (ScoreEntity score in newScores) { - insertionScript.AppendLine($"""insert into Scores values('{score.ScoreId}', '{score.SlotId}', '{score.Type}', '', '{score.Points}', '{score.ChildSlotId}', '{score.Timestamp}', '{score.UserId}');"""); + insertionScript.AppendLine($""" + insert into Scores (ScoreId, SlotId, Type, Points, ChildSlotId, Timestamp, UserId) + values('{score.ScoreId}', '{score.SlotId}', '{score.Type}', '', '{score.Points}', '{score.ChildSlotId}', '{score.Timestamp}', '{score.UserId}'); + """); } await database.Database.ExecuteSqlRawAsync(insertionScript.ToString()); @@ -160,6 +137,7 @@ public override async Task Run(DatabaseContext database) // Get all slots with at least 1 score List slots = await GetAllSlots(database); List scores = await GetAllScores(database); + // Don't run migration if there are no scores if (scores == null || scores.Count == 0) return true; @@ -200,7 +178,7 @@ public override async Task Run(DatabaseContext database) } /// - /// + /// This function takes in a list of scores and creates a map of players and their highest score /// private static Dictionary CreateHighestScores(List scores, IReadOnlyCollection userCache) { diff --git a/ProjectLighthouse/Helpers/MigrationHelper.cs b/ProjectLighthouse/Helpers/MigrationHelper.cs new file mode 100644 index 000000000..e1346d135 --- /dev/null +++ b/ProjectLighthouse/Helpers/MigrationHelper.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; + +namespace LBPUnion.ProjectLighthouse.Helpers; + +public static class MigrationHelper +{ + public static async Task> GetAllObjects(DbContext database, string commandText, Func returnFunc) + { + DbConnection dbConnection = database.Database.GetDbConnection(); + + await using DbCommand cmd = dbConnection.CreateCommand(); + cmd.CommandText = commandText; + cmd.Transaction = GetDbTransaction(database.Database.CurrentTransaction); + + await using DbDataReader reader = await cmd.ExecuteReaderAsync(); + List items = new(); + + if (!reader.HasRows) return default; + + while (await reader.ReadAsync()) + { + items.Add(returnFunc(reader)); + } + + return items; + } + + private static DbTransaction GetDbTransaction(IDbContextTransaction dbContextTransaction) + { + if (dbContextTransaction is not IInfrastructure accessor) + { + throw new InvalidOperationException(RelationalStrings.RelationalNotInUse); + } + + return accessor.GetInfrastructure(); + } +} \ No newline at end of file diff --git a/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs b/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs index 2a3b82c75..93aedc9ca 100644 --- a/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs +++ b/ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs @@ -8,22 +8,22 @@ public class ScoreEntity { [Key] public int ScoreId { get; set; } - + public int SlotId { get; set; } - + [ForeignKey(nameof(SlotId))] public SlotEntity Slot { get; set; } - + public int ChildSlotId { get; set; } - + public int Type { get; set; } public int UserId { get; set; } - + [ForeignKey(nameof(UserId))] public UserEntity User { get; set; } public int Points { get; set; } - + public long Timestamp { get; set; } } \ No newline at end of file From da82f486795c03c364aedd3675c6ad9f5d4d9d45 Mon Sep 17 00:00:00 2001 From: Slendy Date: Thu, 20 Jul 2023 19:56:00 -0500 Subject: [PATCH 4/8] Make Score Timestamp default the current time --- .../MigrationTasks/SwitchScoreToUserIdMigration.cs | 4 ---- .../Migrations/20230628043618_AddUserIdAndTimestampToScore.cs | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs index 2f846161d..6b7e43581 100644 --- a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs +++ b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.ComponentModel.DataAnnotations.Schema; using System.Data; -using System.Data.Common; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -12,9 +11,6 @@ using LBPUnion.ProjectLighthouse.Types.Entities.Level; using LBPUnion.ProjectLighthouse.Types.Maintenance; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Diagnostics; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Storage; namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks; diff --git a/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs b/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs index 15c38f923..727964fa4 100644 --- a/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs +++ b/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs @@ -1,4 +1,5 @@ using LBPUnion.ProjectLighthouse.Database; +using LBPUnion.ProjectLighthouse.Helpers; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; @@ -17,7 +18,7 @@ protected override void Up(MigrationBuilder migrationBuilder) table: "Scores", type: "bigint", nullable: false, - defaultValue: 0L); + defaultValue: TimeHelper.TimestampMillis); migrationBuilder.AddColumn( name: "UserId", From 080003239c91a013adb0a57d1acf25c77f4a5e91 Mon Sep 17 00:00:00 2001 From: Slendy Date: Thu, 20 Jul 2023 20:25:06 -0500 Subject: [PATCH 5/8] Chunk insertions to reduce packet size and give all scores the same Timestamp --- .../Maintenance/MaintenanceHelper.cs | 2 +- .../SwitchScoreToUserIdMigration.cs | 20 ++++++++++++------- ...0628043618_AddUserIdAndTimestampToScore.cs | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs b/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs index f87252a0f..eac0f38ec 100644 --- a/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs +++ b/ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs @@ -112,7 +112,7 @@ public static async Task RunMigration(DatabaseContext database, MigrationT } stopwatch.Stop(); - Logger.Success($"Successfully completed LH migration {migrationTask.Name()} in {stopwatch.ElapsedMilliseconds}", LogArea.Database); + Logger.Success($"Successfully completed LH migration {migrationTask.Name()} in {stopwatch.ElapsedMilliseconds}ms", LogArea.Database); CompletedMigrationEntity completedMigration = new() { diff --git a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs index 6b7e43581..7c7526374 100644 --- a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs +++ b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs @@ -110,17 +110,22 @@ private static async Task ApplyFixedScores(DatabaseContext database, IReadOnlyLi // Delete all existing scores await database.Scores.ExecuteDeleteAsync(); - StringBuilder insertionScript = new(); + long timestamp = TimeHelper.TimestampMillis; + // This is significantly faster than using standard EntityFramework Add and Save albeit a little wacky - foreach (ScoreEntity score in newScores) + foreach (ScoreEntity[] scoreChunk in newScores.Chunk(50_000)) { - insertionScript.AppendLine($""" + StringBuilder insertionScript = new(); + foreach (ScoreEntity score in scoreChunk) + { + insertionScript.AppendLine($""" insert into Scores (ScoreId, SlotId, Type, Points, ChildSlotId, Timestamp, UserId) - values('{score.ScoreId}', '{score.SlotId}', '{score.Type}', '', '{score.Points}', '{score.ChildSlotId}', '{score.Timestamp}', '{score.UserId}'); + values('{score.ScoreId}', '{score.SlotId}', '{score.Type}', '{score.Points}', '{score.ChildSlotId}', '{timestamp}', '{score.UserId}'); """); - } + } - await database.Database.ExecuteSqlRawAsync(insertionScript.ToString()); + await database.Database.ExecuteSqlRawAsync(insertionScript.ToString()); + } } public override async Task Run(DatabaseContext database) @@ -213,7 +218,8 @@ from kvp in highestScores SlotId = slot.SlotId, ChildSlotId = slotGroup.Key, Points = kvp.Value, - Timestamp = TimeHelper.TimestampMillis, + // This gets set before insertion + Timestamp = 0L, Type = scoreType, }).ToList(); } diff --git a/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs b/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs index 727964fa4..51c82f643 100644 --- a/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs +++ b/ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs @@ -18,7 +18,7 @@ protected override void Up(MigrationBuilder migrationBuilder) table: "Scores", type: "bigint", nullable: false, - defaultValue: TimeHelper.TimestampMillis); + defaultValue: 0L); migrationBuilder.AddColumn( name: "UserId", From cb65a5686078083ab83eed606ebc6e04a08e8db4 Mon Sep 17 00:00:00 2001 From: Slendy Date: Thu, 20 Jul 2023 20:34:55 -0500 Subject: [PATCH 6/8] Fix serialization of GameScore --- ProjectLighthouse/Types/Serialization/GameScore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjectLighthouse/Types/Serialization/GameScore.cs b/ProjectLighthouse/Types/Serialization/GameScore.cs index dc33a6333..1195ce96f 100644 --- a/ProjectLighthouse/Types/Serialization/GameScore.cs +++ b/ProjectLighthouse/Types/Serialization/GameScore.cs @@ -10,7 +10,7 @@ namespace LBPUnion.ProjectLighthouse.Types.Serialization; [XmlRoot("playRecord")] [XmlType("playRecord")] -public class GameScore : ILbpSerializable +public class GameScore : ILbpSerializable, INeedsPreparationForSerialization { [XmlIgnore] public int UserId { get; set; } From d50400e52151dea7b251e992e02711686836a0bf Mon Sep 17 00:00:00 2001 From: Slendy Date: Thu, 20 Jul 2023 20:44:43 -0500 Subject: [PATCH 7/8] Break score ties by time then scoreId --- .../Controllers/Slots/ScoreController.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs b/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs index dd1ffdb40..92bff61bb 100644 --- a/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs +++ b/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs @@ -285,19 +285,20 @@ private async Task GetScores(LeaderboardOptions options) .Select(s => new { Score = s, - Rank = scoreQuery.Count(s2 => s2.Points > s.Points)+1, + Rank = scoreQuery.Count(s2 => s2.Points > s.Points) + 1, }).FirstOrDefaultAsync(); int skipAmt = options.PageStart != -1 || myScore == null ? options.PageStart - 1 : myScore.Rank - 3; var rankedScores = scoreQuery.OrderByDescending(s => s.Points) + .ThenBy(s => s.Timestamp) .ThenBy(s => s.ScoreId) .Skip(Math.Max(0, skipAmt)) .Take(Math.Min(options.PageSize, 30)) .Select(s => new { Score = s, - Rank = scoreQuery.Count(s2 => s2.Points > s.Points)+1, + Rank = scoreQuery.Count(s2 => s2.Points > s.Points) + 1, }) .ToList(); From c52761b960f1cff25d9ed5ff1fd5028b1656bead Mon Sep 17 00:00:00 2001 From: Slendy Date: Wed, 2 Aug 2023 00:34:30 -0500 Subject: [PATCH 8/8] Make lighthouse score migration not dependent on current score implementation --- .../Controllers/Slots/ScoreController.cs | 10 +-- .../SwitchScoreToUserIdMigration.cs | 74 ++++++++++--------- 2 files changed, 45 insertions(+), 39 deletions(-) diff --git a/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs b/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs index 92bff61bb..544e57994 100644 --- a/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs +++ b/ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs @@ -1,5 +1,4 @@ #nullable enable -using System.Diagnostics.CodeAnalysis; using LBPUnion.ProjectLighthouse.Database; using LBPUnion.ProjectLighthouse.Extensions; using LBPUnion.ProjectLighthouse.Helpers; @@ -34,11 +33,10 @@ private static int[] GetFriendIds(int userId) { UserFriendData? store = UserFriendStore.GetUserFriendData(userId); List? friendIds = store?.FriendIds; - friendIds ??= new List - { - userId, - }; - return friendIds.Append(userId).Distinct().ToArray(); + friendIds ??= new List(); + friendIds.Add(userId); + + return friendIds.Distinct().ToArray(); } [HttpPost("scoreboard/{slotType}/{id:int}")] diff --git a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs index 7c7526374..03fba51ab 100644 --- a/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs +++ b/ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs @@ -8,7 +8,6 @@ using System.Threading.Tasks; using LBPUnion.ProjectLighthouse.Database; using LBPUnion.ProjectLighthouse.Helpers; -using LBPUnion.ProjectLighthouse.Types.Entities.Level; using LBPUnion.ProjectLighthouse.Types.Maintenance; using Microsoft.EntityFrameworkCore; @@ -18,7 +17,7 @@ public class SwitchScoreToUserIdMigration : MigrationTask { #region DB entity replication stuff - private class MigrationScore + private class PostMigrationScore { public int ScoreId { get; set; } @@ -28,17 +27,29 @@ private class MigrationScore public int Type { get; set; } - public string PlayerIdCollection { get; set; } - - [NotMapped] - public IEnumerable PlayerIds => this.PlayerIdCollection.Split(","); + public int UserId { get; set; } public int Points { get; set; } + + public long Timestamp { get; set; } } - private class MigrationSlot + private class PreMigrationScore { + public int ScoreId { get; set; } + public int SlotId { get; set; } + + public int ChildSlotId { get; set; } + + public int Type { get; set; } + + public string PlayerIdCollection { get; set; } + + [NotMapped] + public IEnumerable PlayerIds => this.PlayerIdCollection.Split(","); + + public int Points { get; set; } } private class MigrationUser @@ -54,11 +65,11 @@ private class MigrationUser public override MigrationHook HookType() => MigrationHook.Before; - private static async Task> GetAllScores(DbContext database) + private static async Task> GetAllScores(DbContext database) { return await MigrationHelper.GetAllObjects(database, "select * from Scores", - reader => new MigrationScore + reader => new PreMigrationScore { ScoreId = reader.GetInt32("ScoreId"), SlotId = reader.GetInt32("SlotId"), @@ -80,21 +91,18 @@ private static async Task> GetAllUsers(DbContext database) }); } - private static async Task> GetAllSlots(DbContext database) + private static async Task> GetAllSlots(DbContext database) { return await MigrationHelper.GetAllObjects(database, "select SlotId from Slots", - reader => new MigrationSlot - { - SlotId = reader.GetInt32("SlotId"), - }); + reader => reader.GetInt32("SlotId")); } /// /// This function deletes all existing scores and inserts the new generated scores /// All scores must be deleted because MySQL doesn't allow you to change primary keys /// - private static async Task ApplyFixedScores(DatabaseContext database, IReadOnlyList newScores) + private static async Task ApplyFixedScores(DatabaseContext database, IReadOnlyList newScores) { // Re-order scores (The order doesn't make any difference but since we're already deleting everything we may as well) newScores = newScores.OrderByDescending(s => s.SlotId) @@ -113,10 +121,10 @@ private static async Task ApplyFixedScores(DatabaseContext database, IReadOnlyLi long timestamp = TimeHelper.TimestampMillis; // This is significantly faster than using standard EntityFramework Add and Save albeit a little wacky - foreach (ScoreEntity[] scoreChunk in newScores.Chunk(50_000)) + foreach (PostMigrationScore[] scoreChunk in newScores.Chunk(50_000)) { StringBuilder insertionScript = new(); - foreach (ScoreEntity score in scoreChunk) + foreach (PostMigrationScore score in scoreChunk) { insertionScript.AppendLine($""" insert into Scores (ScoreId, SlotId, Type, Points, ChildSlotId, Timestamp, UserId) @@ -134,22 +142,22 @@ public override async Task Run(DatabaseContext database) { 1, 2, 3, 4, 7, }; - ConcurrentBag newScores = new(); - // Get all slots with at least 1 score - List slots = await GetAllSlots(database); - List scores = await GetAllScores(database); + ConcurrentBag newScores = new(); + + List slotIds = await GetAllSlots(database); + List scores = await GetAllScores(database); // Don't run migration if there are no scores if (scores == null || scores.Count == 0) return true; List users = await GetAllUsers(database); - ConcurrentQueue<(MigrationSlot slot, int type)> collection = new(); - foreach (MigrationSlot slot in slots.Where(s => scores.Any(score => s.SlotId == score.SlotId))) + ConcurrentQueue<(int slotId, int type)> collection = new(); + foreach (int slotId in slotIds.Where(id => scores.Any(score => id == score.SlotId))) { foreach (int type in scoreTypes) { - collection.Enqueue((slot, type)); + collection.Enqueue((slotId, type)); } } @@ -158,11 +166,11 @@ public override async Task Run(DatabaseContext database) { Task task = Task.Run(() => { - while (collection.TryDequeue(out (MigrationSlot slot, int type) item)) + while (collection.TryDequeue(out (int slotId, int type) item)) { - List fixedScores = FixScores(users, - item.slot, - scores.Where(s => s.SlotId == item.slot.SlotId).Where(s => s.Type == item.type).ToList(), + List fixedScores = FixScores(users, + item.slotId, + scores.Where(s => s.SlotId == item.slotId).Where(s => s.Type == item.type).ToList(), item.type) .ToList(); fixedScores.AsParallel().ForAll(score => newScores.Add(score)); @@ -181,10 +189,10 @@ public override async Task Run(DatabaseContext database) /// /// This function takes in a list of scores and creates a map of players and their highest score /// - private static Dictionary CreateHighestScores(List scores, IReadOnlyCollection userCache) + private static Dictionary CreateHighestScores(List scores, IReadOnlyCollection userCache) { Dictionary maxPointsByPlayer = new(StringComparer.InvariantCultureIgnoreCase); - foreach (MigrationScore score in scores) + foreach (PreMigrationScore score in scores) { IEnumerable players = score.PlayerIds; foreach (string player in players) @@ -205,17 +213,17 @@ private static Dictionary CreateHighestScores(List /// This function groups slots by ChildSlotId to account for adventure scores and then for each user /// finds their highest score on that level and adds a new Score /// - private static IEnumerable FixScores(IReadOnlyCollection userCache, MigrationSlot slot, IEnumerable scores, int scoreType) + private static IEnumerable FixScores(IReadOnlyCollection userCache, int slotId, IEnumerable scores, int scoreType) { return ( from slotGroup in scores.GroupBy(s => s.ChildSlotId) let highestScores = CreateHighestScores(slotGroup.ToList(), userCache) from kvp in highestScores let userId = userCache.Where(u => u.Username == kvp.Key).Select(u => u.UserId).First() - select new ScoreEntity + select new PostMigrationScore { UserId = userId, - SlotId = slot.SlotId, + SlotId = slotId, ChildSlotId = slotGroup.Key, Points = kvp.Value, // This gets set before insertion