From 5643f4004a4f58c1e7a220837eba2441ba29a8b8 Mon Sep 17 00:00:00 2001 From: "George Njeri (Swagfin)" Date: Fri, 27 Jun 2025 23:32:35 +0300 Subject: [PATCH 1/6] chore: refactoring and sorting issues with in-depth delete --- SemanticBackup.Core/Helpers/WithRetry.cs | 60 +++++++++++++++++++ .../Interfaces/IBackupRecordRepository.cs | 2 +- .../IContentDeliveryRecordRepository.cs | 2 +- .../BackgroundJobs/BackupBackgroundJob.cs | 38 ++++++------ ...ckupRecordDeliveryDispatchBackgroundJob.cs | 41 +------------ .../BackupRecordRepositoryLiteDb.cs | 59 +++++++----------- .../ContentDeliveryRecordRepositoryLiteDb.cs | 10 ++-- 7 files changed, 107 insertions(+), 105 deletions(-) create mode 100644 SemanticBackup.Core/Helpers/WithRetry.cs diff --git a/SemanticBackup.Core/Helpers/WithRetry.cs b/SemanticBackup.Core/Helpers/WithRetry.cs new file mode 100644 index 0000000..66d1788 --- /dev/null +++ b/SemanticBackup.Core/Helpers/WithRetry.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace SemanticBackup.Core.Helpers +{ + public static class WithRetry + { + /// + /// Retries an asynchronous function that returns a result (Task<T>). + /// + public static async Task TaskAsync(Func> operation, int maxRetries = 2, TimeSpan? delay = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(operation); + ArgumentOutOfRangeException.ThrowIfNegative(maxRetries); + + delay ??= TimeSpan.FromSeconds(1); + + for (int attempt = 1; ; attempt++) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + return await operation().ConfigureAwait(false); + } + catch (Exception ex) when (attempt < maxRetries) + { + Console.WriteLine($"[Attempt {attempt}] Failed, Error: {ex.Message}"); + await Task.Delay(delay.Value, cancellationToken).ConfigureAwait(false); + } + } + } + + /// + /// Retries an asynchronous function that does not return a result (Task). + /// + public static async Task TaskAsync(Func operation, int maxRetries = 2, TimeSpan? delay = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(operation); + ArgumentOutOfRangeException.ThrowIfNegative(maxRetries); + + delay ??= TimeSpan.FromSeconds(1); + + for (int attempt = 1; ; attempt++) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + await operation().ConfigureAwait(false); + return; + } + catch (Exception ex) when (attempt < maxRetries) + { + Console.WriteLine($"[Attempt {attempt}] Failed, Error: {ex.Message}"); + await Task.Delay(delay.Value, cancellationToken).ConfigureAwait(false); + } + } + } + } +} diff --git a/SemanticBackup.Core/Interfaces/IBackupRecordRepository.cs b/SemanticBackup.Core/Interfaces/IBackupRecordRepository.cs index 9703017..78cc254 100644 --- a/SemanticBackup.Core/Interfaces/IBackupRecordRepository.cs +++ b/SemanticBackup.Core/Interfaces/IBackupRecordRepository.cs @@ -9,9 +9,9 @@ public interface IBackupRecordRepository { Task> GetAllAsync(string resourceGroupId); Task GetByIdAsync(long id); - Task RemoveAsync(long id); Task AddOrUpdateAsync(BackupRecord record); Task UpdateAsync(BackupRecord record); + Task RemoveAsync(long id); Task UpdateStatusFeedAsync(long id, string status, string message = null, long executionInMilliseconds = 0, string newFilePath = null); Task UpdateRestoreStatusFeedAsync(long id, string status, string message = null, string confirmationToken = null); Task> GetAllByStatusAsync(string status); diff --git a/SemanticBackup.Core/Interfaces/IContentDeliveryRecordRepository.cs b/SemanticBackup.Core/Interfaces/IContentDeliveryRecordRepository.cs index f75fb81..a351cd0 100644 --- a/SemanticBackup.Core/Interfaces/IContentDeliveryRecordRepository.cs +++ b/SemanticBackup.Core/Interfaces/IContentDeliveryRecordRepository.cs @@ -6,9 +6,9 @@ namespace SemanticBackup.Core.Interfaces { public interface IContentDeliveryRecordRepository { - Task RemoveAsync(string id); Task AddOrUpdateAsync(BackupRecordDelivery record); Task UpdateStatusFeedAsync(string id, string status, string message = null, long executionInMilliseconds = 0); + Task RemoveAsync(string id); Task> GetAllByStatusAsync(string status); Task> GetAllByBackupRecordIdAsync(long id); Task> GetAllNoneResponsiveAsync(List statusChecks, int minuteDifference); diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs index 215a2d2..40f362f 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs @@ -103,9 +103,8 @@ private void SetupBackgroundService(CancellationToken cancellationToken) } } //Check if Any Delete - if (scheduleToDelete.Count > 0) - foreach (var rm in scheduleToDelete) - await backupRecordPersistanceService.RemoveAsync(rm); + foreach (var rm in scheduleToDelete) + await backupRecordPersistanceService.RemoveAsync(rm); } } @@ -124,26 +123,25 @@ private void SetupBackgroundRemovedExpiredBackupsService(CancellationToken cance { while (!cancellationToken.IsCancellationRequested) { - await Task.Delay(3000); //Runs After 3sec + await Task.Delay(3000, cancellationToken); //Runs After 3sec try { using IServiceScope scope = _serviceScopeFactory.CreateScope(); //DI Injections IBackupRecordRepository backupRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); + IContentDeliveryRecordRepository contentDeliveryRecordsService = scope.ServiceProvider.GetRequiredService(); //Proceed - List expiredBackups = await backupRecordPersistanceService.GetAllExpiredAsync(); - if (expiredBackups != null && expiredBackups.Count > 0) + List expiredBackups = (await backupRecordPersistanceService.GetAllExpiredAsync()) ?? []; + //proceed + foreach (BackupRecord rm in expiredBackups.Take(50).ToList()) { - foreach (BackupRecord rm in expiredBackups.Take(50).ToList()) - if (!await backupRecordPersistanceService.RemoveAsync(rm.Id)) - _logger.LogWarning($"Unable to delete Expired Backup Record: {rm.Id}"); - else - { - _logger.LogInformation($"Removed Expired Backup Record, Id: {rm.Id}"); - //Options InDepth Delete - if (_persistanceOptions.InDepthBackupRecordDeleteEnabled) - await StartInDepthDeleteForAsync(rm); - } + //get relation + List rmBackupRecords = (await contentDeliveryRecordsService.GetAllByBackupRecordIdAsync(rm.Id)) ?? []; + //remove with file + await backupRecordPersistanceService.RemoveAsync(rm.Id); + //Options InDepth Delete + if (_persistanceOptions.InDepthBackupRecordDeleteEnabled) + await StartInDepthDeleteForAsync(rm, rmBackupRecords); } } catch (Exception ex) @@ -155,7 +153,7 @@ private void SetupBackgroundRemovedExpiredBackupsService(CancellationToken cance t.Start(); } - private async Task StartInDepthDeleteForAsync(BackupRecord rm) + private async Task StartInDepthDeleteForAsync(BackupRecord rm, List rmBackupRecords) { try { @@ -163,7 +161,6 @@ private async Task StartInDepthDeleteForAsync(BackupRecord rm) //scope using IServiceScope scope = _serviceScopeFactory.CreateScope(); IResourceGroupRepository resourceGroupPersistanceService = scope.ServiceProvider.GetRequiredService(); - IContentDeliveryRecordRepository contentDeliveryRecordsService = scope.ServiceProvider.GetRequiredService(); BotsManagerBackgroundJob botsManagerBackgroundJob = scope.ServiceProvider.GetRequiredService(); IDatabaseInfoRepository databaseInfoRepository = scope.ServiceProvider.GetRequiredService(); //get db information @@ -173,11 +170,10 @@ private async Task StartInDepthDeleteForAsync(BackupRecord rm) if (resourceGroup == null) return; //Proceed - List dbRecords = await contentDeliveryRecordsService.GetAllByBackupRecordIdAsync(rm.Id); //database record content delivery - if (dbRecords == null) + if (rmBackupRecords == null) return; List supportedInDepthDelete = [BackupDeliveryConfigTypes.Dropbox.ToString(), BackupDeliveryConfigTypes.AzureBlobStorage.ToString()]; - List supportedDeliveryRecords = [.. dbRecords.Where(x => supportedInDepthDelete.Contains(x.DeliveryType))]; + List supportedDeliveryRecords = [.. rmBackupRecords.Where(x => supportedInDepthDelete.Contains(x.DeliveryType))]; if (supportedDeliveryRecords == null || supportedDeliveryRecords.Count == 0) return; foreach (BackupRecordDelivery rec in supportedDeliveryRecords) diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs index 10604f4..e418e2d 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs @@ -134,9 +134,8 @@ private void SetupBackgroundService(CancellationToken cancellationToken) } } //Check if Any Delete - if (scheduleToDeleteRecords.Count > 0) - foreach (var rm in scheduleToDeleteRecords) - await contentDeliveryRecordPersistanceService.RemoveAsync(rm); + foreach (string rm in scheduleToDeleteRecords) + await contentDeliveryRecordPersistanceService.RemoveAsync(rm); } } catch (Exception ex) @@ -148,41 +147,5 @@ private void SetupBackgroundService(CancellationToken cancellationToken) }); t.Start(); } - - private void SetupBackgroundRemovedExpiredBackupsService(CancellationToken cancellationToken) - { - Thread t = new Thread(async () => - { - while (!cancellationToken.IsCancellationRequested) - { - await Task.Delay(60000, cancellationToken); //Runs After 1 Minute - try - { - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - //DI INJECTIONS - IBackupRecordRepository backupRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); - //Proceed - List expiredBackups = await backupRecordPersistanceService.GetAllExpiredAsync(); - if (expiredBackups != null && expiredBackups.Count > 0) - { - List toDeleteList = new(); - foreach (BackupRecord backupRecord in expiredBackups) - toDeleteList.Add(backupRecord.Id); - _logger.LogInformation($"Queued ({expiredBackups.Count}) Expired Records for Delete"); - //Check if Any Delete - if (toDeleteList.Count > 0) - foreach (var rm in toDeleteList) - if (!(await backupRecordPersistanceService.RemoveAsync(rm))) - _logger.LogWarning("Unable to delete Expired Backup Record"); - } - } - catch (Exception ex) - { - _logger.LogError(ex.Message); - } - } - }); - t.Start(); - } } } diff --git a/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs b/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs index 3d44ae8..aff0df9 100644 --- a/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs +++ b/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs @@ -1,13 +1,13 @@ using LiteDB; using LiteDB.Async; using SemanticBackup.Core; +using SemanticBackup.Core.Helpers; using SemanticBackup.Core.Interfaces; using SemanticBackup.Core.Models; using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Threading; using System.Threading.Tasks; namespace SemanticBackup.Infrastructure.Implementations @@ -154,25 +154,22 @@ public async Task VerifyBackupRecordInResourceGroupThrowIfNotExist _ = await _databaseInfoRepository.VerifyDatabaseInResourceGroupThrowIfNotExistAsync(resourceGroupId, backupRecordResponse.BackupDatabaseInfoId ?? string.Empty); return backupRecordResponse; } - public async Task RemoveAsync(long id) + + public async Task RemoveAsync(long id) { var collection = _db.GetCollection(); var objFound = await collection.Query().Where(x => x.Id == id).FirstOrDefaultAsync(); if (objFound != null) { string pathToRemove = objFound.Path; - bool removedSuccess = await collection.DeleteAsync(new BsonValue(objFound.Id)); - if (removedSuccess) - { - TryDeleteContentDispatchRecordsAsync(id); - TryDeleteOldFile(pathToRemove); - } - return removedSuccess; + await collection.DeleteAsync(new BsonValue(objFound.Id)); + //remove content dispatch records + await TryDeleteContentDispatchRecordsAsync(id); + await TryDeleteOldFileAsync(pathToRemove); } - return false; } - private async void TryDeleteContentDispatchRecordsAsync(long id) + private async Task TryDeleteContentDispatchRecordsAsync(long id) { try { @@ -197,8 +194,8 @@ public async Task> GetAllReadyAndPendingDeliveryAsync() } public async Task UpdateDeliveryRunnedAsync(long backupRecordId, bool hasRun, string executedDeliveryRunStatus) { - var collection = _db.GetCollection(); - var objFound = await collection.Query().Where(x => x.Id == backupRecordId).FirstOrDefaultAsync(); + ILiteCollectionAsync collection = _db.GetCollection(); + BackupRecord objFound = await collection.Query().Where(x => x.Id == backupRecordId).FirstOrDefaultAsync(); if (objFound != null) { objFound.ExecutedDeliveryRun = hasRun; @@ -210,6 +207,7 @@ public async Task UpdateDeliveryRunnedAsync(long backupRecordId, bool hasR } return false; } + private void DispatchUpdatedStatus(BackupRecord record, bool isNewRecord = false) { if (_backupRecordStatusChangedNotifiers != null) @@ -220,36 +218,21 @@ private void DispatchUpdatedStatus(BackupRecord record, bool isNewRecord = false } catch { } } - private void TryDeleteOldFile(string path) + + private static async Task TryDeleteOldFileAsync(string path) { try { - bool success = false; - int attempts = 0; - do - { - try - { - attempts++; - if (File.Exists(path)) - File.Delete(path); - success = true; - } - catch (Exception ex) - { - if (attempts >= 10) - { - Thread.Sleep(2000); - throw new Exception($"Maximum Deletion Attempts, Error: {ex.Message}"); - } - } - } - while (!success); + await WithRetry.TaskAsync(() => + { + //check file exists + if (File.Exists(path)) + File.Delete(path); + return Task.CompletedTask; + }, 3, TimeSpan.FromSeconds(5)); } - catch (Exception) { } + catch (Exception ex) { Console.WriteLine($"Failed to remove File: {ex.Message}"); } } - - } } diff --git a/SemanticBackup.Infrastructure/Implementations/ContentDeliveryRecordRepositoryLiteDb.cs b/SemanticBackup.Infrastructure/Implementations/ContentDeliveryRecordRepositoryLiteDb.cs index 266cc6b..f65ed8a 100644 --- a/SemanticBackup.Infrastructure/Implementations/ContentDeliveryRecordRepositoryLiteDb.cs +++ b/SemanticBackup.Infrastructure/Implementations/ContentDeliveryRecordRepositoryLiteDb.cs @@ -76,15 +76,15 @@ public async Task UpdateStatusFeedAsync(string id, string status, string m } return false; } - public async Task RemoveAsync(string id) + + public async Task RemoveAsync(string id) { - var collection = _db.GetCollection(); - var objFound = await collection.Query().Where(x => x.Id == id).FirstOrDefaultAsync(); + ILiteCollectionAsync collection = _db.GetCollection(); + BackupRecordDelivery objFound = await collection.Query().Where(x => x.Id == id).FirstOrDefaultAsync(); if (objFound != null) { - return await collection.DeleteAsync(new BsonValue(objFound.Id)); + await collection.DeleteAsync(new BsonValue(objFound.Id)); } - return false; } public async Task> GetAllNoneResponsiveAsync(List statusChecks, int minuteDifference) From b30339e745ff3ff2d92ebf1225998016672005ea Mon Sep 17 00:00:00 2001 From: "George Njeri (Swagfin)" Date: Fri, 27 Jun 2025 23:58:26 +0300 Subject: [PATCH 2/6] chore: additional refactoring --- .../Interfaces/IBackupRecordRepository.cs | 2 +- .../BackgroundJobs/BackupBackgroundJob.cs | 88 ++++++++++--------- .../Bots/InDepthDeleteDropboxBot.cs | 16 +--- .../Bots/InDepthDeleteObjectStorageBot.cs | 18 ++-- .../BackgroundJobs/Bots/MySQLBackupBot.cs | 28 ++---- .../BackgroundJobs/Bots/SQLBackupBot.cs | 30 +++---- .../BackupRecordRepositoryLiteDb.cs | 2 +- .../ResourceGroupRepositoryLiteDb.cs | 2 +- 8 files changed, 80 insertions(+), 106 deletions(-) diff --git a/SemanticBackup.Core/Interfaces/IBackupRecordRepository.cs b/SemanticBackup.Core/Interfaces/IBackupRecordRepository.cs index 78cc254..0d51d22 100644 --- a/SemanticBackup.Core/Interfaces/IBackupRecordRepository.cs +++ b/SemanticBackup.Core/Interfaces/IBackupRecordRepository.cs @@ -11,7 +11,7 @@ public interface IBackupRecordRepository Task GetByIdAsync(long id); Task AddOrUpdateAsync(BackupRecord record); Task UpdateAsync(BackupRecord record); - Task RemoveAsync(long id); + Task RemoveWithFileAsync(long id); Task UpdateStatusFeedAsync(long id, string status, string message = null, long executionInMilliseconds = 0, string newFilePath = null); Task UpdateRestoreStatusFeedAsync(long id, string status, string message = null, string confirmationToken = null); Task> GetAllByStatusAsync(string status); diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs index 40f362f..6585fef 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs @@ -1,5 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using SemanticBackup.Core; using SemanticBackup.Core.Interfaces; @@ -17,15 +16,38 @@ public class BackupBackgroundJob : IHostedService { private readonly ILogger _logger; private readonly SystemConfigOptions _persistanceOptions; - private readonly IServiceScopeFactory _serviceScopeFactory; private readonly BotsManagerBackgroundJob _botsManagerBackgroundJob; - public BackupBackgroundJob(ILogger logger, SystemConfigOptions persistanceOptions, IServiceScopeFactory serviceScopeFactory, BotsManagerBackgroundJob botsManagerBackgroundJob) + private readonly IBackupProviderForSQLServer _providerForSQLServer; + private readonly IBackupProviderForMySQLServer _providerForMySqlServer; + + private readonly IResourceGroupRepository _resourceGroupRepository; + private readonly IBackupRecordRepository _backupRecordRepository; + private readonly IContentDeliveryRecordRepository _deliveryRecordRepository; + private readonly IDatabaseInfoRepository _databaseInfoRepository; + + public BackupBackgroundJob( + ILogger logger, + SystemConfigOptions persistanceOptions, + BotsManagerBackgroundJob botsManagerBackgroundJob, + IBackupProviderForSQLServer providerForSQLServer, + IBackupProviderForMySQLServer providerForMySqlServer, + + IResourceGroupRepository resourceGroupRepository, + IBackupRecordRepository backupRecordRepository, + IContentDeliveryRecordRepository contentDeliveryRecordRepository, + IDatabaseInfoRepository databaseInfoRepository + ) { _logger = logger; _persistanceOptions = persistanceOptions; - _serviceScopeFactory = serviceScopeFactory; _botsManagerBackgroundJob = botsManagerBackgroundJob; + _providerForSQLServer = providerForSQLServer; + _providerForMySqlServer = providerForMySqlServer; + _resourceGroupRepository = resourceGroupRepository; + _backupRecordRepository = backupRecordRepository; + _deliveryRecordRepository = contentDeliveryRecordRepository; + _databaseInfoRepository = databaseInfoRepository; } public Task StartAsync(CancellationToken cancellationToken) @@ -48,23 +70,18 @@ private void SetupBackgroundService(CancellationToken cancellationToken) while (!cancellationToken.IsCancellationRequested) { //Await - await Task.Delay(5000, cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); try { - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - //DI Injections - IBackupRecordRepository backupRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); - IDatabaseInfoRepository databaseInfoPersistanceService = scope.ServiceProvider.GetRequiredService(); - IResourceGroupRepository resourceGroupPersistanceService = scope.ServiceProvider.GetRequiredService(); //Proceed - List queuedBackups = await backupRecordPersistanceService.GetAllByStatusAsync(BackupRecordStatus.QUEUED.ToString()); + List queuedBackups = await _backupRecordRepository.GetAllByStatusAsync(BackupRecordStatus.QUEUED.ToString()); if (queuedBackups != null && queuedBackups.Count > 0) { List scheduleToDelete = []; foreach (BackupRecord backupRecord in queuedBackups.OrderBy(x => x.RegisteredDateUTC).ToList()) { _logger.LogInformation($"Processing Queued Backup Record Key: #{backupRecord.Id}..."); - BackupDatabaseInfo backupDatabaseInfo = await databaseInfoPersistanceService.GetByIdAsync(backupRecord.BackupDatabaseInfoId); + BackupDatabaseInfo backupDatabaseInfo = await _databaseInfoRepository.GetByIdAsync(backupRecord.BackupDatabaseInfoId); if (backupDatabaseInfo == null) { _logger.LogWarning($"No Database Info matches with Id: {backupRecord.BackupDatabaseInfoId}, Backup Database Record will be Deleted: {backupRecord.Id}"); @@ -73,7 +90,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) else { //Check if valid Resource Group - ResourceGroup resourceGroup = await resourceGroupPersistanceService.GetByIdOrKeyAsync(backupDatabaseInfo.ResourceGroupId); + ResourceGroup resourceGroup = await _resourceGroupRepository.GetByIdOrKeyAsync(backupDatabaseInfo.ResourceGroupId); if (resourceGroup == null) { _logger.LogWarning($"The Database Id: {backupRecord.BackupDatabaseInfoId}, doesn't seem to have been assigned to a valid Resource Group Id: {backupDatabaseInfo.ResourceGroupId}, Record will be Deleted"); @@ -84,13 +101,13 @@ private void SetupBackgroundService(CancellationToken cancellationToken) if (_botsManagerBackgroundJob.HasAvailableResourceGroupBotsCount(resourceGroup.Id, resourceGroup.MaximumRunningBots)) { if (resourceGroup.DbType.Contains("SQLSERVER")) - _botsManagerBackgroundJob.AddBot(new SQLBackupBot(backupDatabaseInfo.DatabaseName, resourceGroup, backupRecord, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new SQLBackupBot(backupDatabaseInfo.DatabaseName, resourceGroup, backupRecord, _providerForSQLServer)); else if (resourceGroup.DbType.Contains("MYSQL") || resourceGroup.DbType.Contains("MARIADB")) - _botsManagerBackgroundJob.AddBot(new MySQLBackupBot(backupDatabaseInfo.DatabaseName, resourceGroup, backupRecord, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new MySQLBackupBot(backupDatabaseInfo.DatabaseName, resourceGroup, backupRecord, _providerForMySqlServer)); else throw new Exception($"No Bot is registered to Handle Database Backups of Type: {resourceGroup.DbType}"); //Finally Update Status - bool updated = await backupRecordPersistanceService.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.EXECUTING.ToString()); + bool updated = await _backupRecordRepository.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.EXECUTING.ToString()); if (updated) _logger.LogInformation($"Processing Queued Backup Record Key: #{backupRecord.Id}...SUCCESS"); else @@ -104,7 +121,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) } //Check if Any Delete foreach (var rm in scheduleToDelete) - await backupRecordPersistanceService.RemoveAsync(rm); + await _backupRecordRepository.RemoveWithFileAsync(rm); } } @@ -123,22 +140,18 @@ private void SetupBackgroundRemovedExpiredBackupsService(CancellationToken cance { while (!cancellationToken.IsCancellationRequested) { - await Task.Delay(3000, cancellationToken); //Runs After 3sec + await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken); try { - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - //DI Injections - IBackupRecordRepository backupRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); - IContentDeliveryRecordRepository contentDeliveryRecordsService = scope.ServiceProvider.GetRequiredService(); //Proceed - List expiredBackups = (await backupRecordPersistanceService.GetAllExpiredAsync()) ?? []; + List expiredBackups = (await _backupRecordRepository.GetAllExpiredAsync()) ?? []; //proceed foreach (BackupRecord rm in expiredBackups.Take(50).ToList()) { //get relation - List rmBackupRecords = (await contentDeliveryRecordsService.GetAllByBackupRecordIdAsync(rm.Id)) ?? []; + List rmBackupRecords = (await _deliveryRecordRepository.GetAllByBackupRecordIdAsync(rm.Id)) ?? []; //remove with file - await backupRecordPersistanceService.RemoveAsync(rm.Id); + await _backupRecordRepository.RemoveWithFileAsync(rm.Id); //Options InDepth Delete if (_persistanceOptions.InDepthBackupRecordDeleteEnabled) await StartInDepthDeleteForAsync(rm, rmBackupRecords); @@ -153,20 +166,15 @@ private void SetupBackgroundRemovedExpiredBackupsService(CancellationToken cance t.Start(); } - private async Task StartInDepthDeleteForAsync(BackupRecord rm, List rmBackupRecords) + private async Task StartInDepthDeleteForAsync(BackupRecord backupRecord, List rmBackupRecords) { try { - if (rm == null) return; - //scope - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - IResourceGroupRepository resourceGroupPersistanceService = scope.ServiceProvider.GetRequiredService(); - BotsManagerBackgroundJob botsManagerBackgroundJob = scope.ServiceProvider.GetRequiredService(); - IDatabaseInfoRepository databaseInfoRepository = scope.ServiceProvider.GetRequiredService(); + if (backupRecord == null) return; //get db information - BackupDatabaseInfo backupRecordDbInfo = await databaseInfoRepository.GetByIdAsync(rm.BackupDatabaseInfoId); + BackupDatabaseInfo backupRecordDbInfo = await _databaseInfoRepository.GetByIdAsync(backupRecord.BackupDatabaseInfoId); //Check if valid Resource Group - ResourceGroup resourceGroup = await resourceGroupPersistanceService.GetByIdOrKeyAsync(backupRecordDbInfo?.ResourceGroupId ?? string.Empty); + ResourceGroup resourceGroup = await _resourceGroupRepository.GetByIdOrKeyAsync(backupRecordDbInfo?.ResourceGroupId ?? string.Empty); if (resourceGroup == null) return; //Proceed @@ -176,17 +184,17 @@ private async Task StartInDepthDeleteForAsync(BackupRecord rm, List supportedDeliveryRecords = [.. rmBackupRecords.Where(x => supportedInDepthDelete.Contains(x.DeliveryType))]; if (supportedDeliveryRecords == null || supportedDeliveryRecords.Count == 0) return; - foreach (BackupRecordDelivery rec in supportedDeliveryRecords) + foreach (BackupRecordDelivery deliveryRecord in supportedDeliveryRecords) { - if (rec.DeliveryType == BackupDeliveryConfigTypes.Dropbox.ToString()) + if (deliveryRecord.DeliveryType == BackupDeliveryConfigTypes.Dropbox.ToString()) { //In Depth Remove From DropBox - botsManagerBackgroundJob.AddBot(new InDepthDeleteDropboxBot(resourceGroup, rm, rec, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new InDepthDeleteDropboxBot(resourceGroup, backupRecord, deliveryRecord)); } - else if (rec.DeliveryType == BackupDeliveryConfigTypes.ObjectStorage.ToString()) + else if (deliveryRecord.DeliveryType == BackupDeliveryConfigTypes.ObjectStorage.ToString()) { //In Depth Remove From Object Storage - botsManagerBackgroundJob.AddBot(new InDepthDeleteObjectStorageBot(resourceGroup, rm, rec, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new InDepthDeleteObjectStorageBot(resourceGroup, backupRecord, deliveryRecord)); } } } diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/InDepthDeleteDropboxBot.cs b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/InDepthDeleteDropboxBot.cs index d722d44..a509fb6 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/InDepthDeleteDropboxBot.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/InDepthDeleteDropboxBot.cs @@ -1,6 +1,4 @@ using Dropbox.Api; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using SemanticBackup.Core.Models; using System; using System.Diagnostics; @@ -15,22 +13,16 @@ internal class InDepthDeleteDropboxBot : IBot private readonly BackupRecordDelivery _contentDeliveryRecord; private readonly ResourceGroup _resourceGroup; private readonly BackupRecord _backupRecord; - private readonly IServiceScopeFactory _scopeFactory; - private readonly ILogger _logger; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; public string BotId => $"{_resourceGroup.Id}::{_backupRecord.Id}::{nameof(InDepthDeleteDropboxBot)}"; public string ResourceGroupId => _resourceGroup.Id; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public InDepthDeleteDropboxBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord, IServiceScopeFactory scopeFactory) + public InDepthDeleteDropboxBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord) { _contentDeliveryRecord = contentDeliveryRecord; _resourceGroup = resourceGroup; _backupRecord = backupRecord; - _scopeFactory = scopeFactory; - //Logger - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); } public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) @@ -39,7 +31,7 @@ public async Task RunAsync(Func _logger; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; public string BotId => $"{_resourceGroup.Id}::{_backupRecord.Id}::{nameof(InDepthDeleteObjectStorageBot)}"; public string ResourceGroupId => _resourceGroup.Id; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public InDepthDeleteObjectStorageBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord, IServiceScopeFactory scopeFactory) + public InDepthDeleteObjectStorageBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord) { _contentDeliveryRecord = contentDeliveryRecord; _resourceGroup = resourceGroup; _backupRecord = backupRecord; - _scopeFactory = scopeFactory; - //Logger - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); } public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) @@ -40,7 +32,7 @@ public async Task RunAsync(Func _logger; + private readonly IBackupProviderForMySQLServer _providerForMySQLServer; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; public string BotId => $"{_resourceGroup.Id}::{_backupRecord.Id}::{nameof(MySQLBackupBot)}"; public string ResourceGroupId => _resourceGroup.Id; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public MySQLBackupBot(string databaseName, ResourceGroup resourceGroup, BackupRecord backupRecord, IServiceScopeFactory serviceScopeFactory) + public MySQLBackupBot(string databaseName, ResourceGroup resourceGroup, BackupRecord backupRecord, IBackupProviderForMySQLServer providerForMySQLServer) { _databaseName = databaseName; _resourceGroup = resourceGroup; _backupRecord = backupRecord; - _scopeFactory = serviceScopeFactory; - //Logger - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); + _providerForMySQLServer = providerForMySQLServer; } + public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) { Status = BotStatus.Starting; Stopwatch stopwatch = new(); try { - _logger.LogInformation("creating backup of Db: {_databaseName}", _databaseName); + Console.WriteLine("creating backup of Db: {_databaseName}", _databaseName); string directory = Path.GetDirectoryName(_backupRecord.Path); if (!Directory.Exists(directory)) Directory.CreateDirectory(directory); @@ -47,12 +42,7 @@ public async Task RunAsync(Func(); - backupedUp = await providerService.BackupDatabaseAsync(_databaseName, _resourceGroup, _backupRecord); - } + bool backupedUp = await _providerForMySQLServer.BackupDatabaseAsync(_databaseName, _resourceGroup, _backupRecord); stopwatch.Stop(); if (!backupedUp) throw new Exception("Creating Backup Failed to Return Success Completion"); @@ -67,13 +57,13 @@ await onDeliveryFeedUpdate(new BackupRecordDeliveryFeed ElapsedMilliseconds = stopwatch.ElapsedMilliseconds }, cancellationToken); - _logger.LogInformation("Successfully Backup of Db: {_databaseName}", _databaseName); + Console.WriteLine("Successfully Backup of Db: {_databaseName}", _databaseName); Status = BotStatus.Completed; } catch (Exception ex) { + Console.WriteLine(ex.Message); Status = BotStatus.Error; - _logger.LogError(ex.Message); stopwatch.Stop(); //notify update await onDeliveryFeedUpdate(new BackupRecordDeliveryFeed diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/SQLBackupBot.cs b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/SQLBackupBot.cs index dbbc1b7..e4153b1 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/SQLBackupBot.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/SQLBackupBot.cs @@ -1,6 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using SemanticBackup.Core.Interfaces; +using SemanticBackup.Core.Interfaces; using SemanticBackup.Core.Models; using System; using System.Diagnostics; @@ -15,30 +13,27 @@ internal class SQLBackupBot : IBot private readonly ResourceGroup _resourceGroup; private readonly string _databaseName; private readonly BackupRecord _backupRecord; - private readonly IServiceScopeFactory _scopeFactory; - private readonly ILogger _logger; + private readonly IBackupProviderForSQLServer _providerForSQLServer; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; public string BotId => $"{_resourceGroup.Id}::{_backupRecord.Id}::{nameof(SQLBackupBot)}"; public string ResourceGroupId => _resourceGroup.Id; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public SQLBackupBot(string databaseName, ResourceGroup resourceGroup, BackupRecord backupRecord, IServiceScopeFactory scopeFactory) + public SQLBackupBot(string databaseName, ResourceGroup resourceGroup, BackupRecord backupRecord, IBackupProviderForSQLServer providerForSQLServer) { _databaseName = databaseName; _resourceGroup = resourceGroup; _backupRecord = backupRecord; - _scopeFactory = scopeFactory; - //Logger - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); + _providerForSQLServer = providerForSQLServer; } + public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) { Status = BotStatus.Starting; Stopwatch stopwatch = new(); try { - _logger.LogInformation("creating backup of Db: {_databaseName}", _databaseName); + Console.WriteLine("creating backup of Db: {_databaseName}", _databaseName); string directory = Path.GetDirectoryName(_backupRecord.Path); if (!Directory.Exists(directory)) Directory.CreateDirectory(directory); @@ -47,12 +42,8 @@ public async Task RunAsync(Func(); - backupedUp = await backupProviderService.BackupDatabaseAsync(_databaseName, _resourceGroup, _backupRecord); - } + bool backupedUp = await _providerForSQLServer.BackupDatabaseAsync(_databaseName, _resourceGroup, _backupRecord); + stopwatch.Stop(); if (!backupedUp) throw new Exception("Creating Backup Failed to Return Success Completion"); @@ -66,13 +57,14 @@ await onDeliveryFeedUpdate(new BackupRecordDeliveryFeed Message = "Successfull", ElapsedMilliseconds = stopwatch.ElapsedMilliseconds }, cancellationToken); - _logger.LogInformation("Successfully Backup of Db: {_databaseName}", _databaseName); + + Console.WriteLine("Successfully Backup of Db: {_databaseName}", _databaseName); Status = BotStatus.Completed; } catch (Exception ex) { Status = BotStatus.Error; - _logger.LogError(ex.Message); + Console.WriteLine(ex.Message); //notify update await onDeliveryFeedUpdate(new BackupRecordDeliveryFeed { diff --git a/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs b/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs index aff0df9..f11fd0d 100644 --- a/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs +++ b/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs @@ -155,7 +155,7 @@ public async Task VerifyBackupRecordInResourceGroupThrowIfNotExist return backupRecordResponse; } - public async Task RemoveAsync(long id) + public async Task RemoveWithFileAsync(long id) { var collection = _db.GetCollection(); var objFound = await collection.Query().Where(x => x.Id == id).FirstOrDefaultAsync(); diff --git a/SemanticBackup.Infrastructure/Implementations/ResourceGroupRepositoryLiteDb.cs b/SemanticBackup.Infrastructure/Implementations/ResourceGroupRepositoryLiteDb.cs index dca3e1d..694f435 100644 --- a/SemanticBackup.Infrastructure/Implementations/ResourceGroupRepositoryLiteDb.cs +++ b/SemanticBackup.Infrastructure/Implementations/ResourceGroupRepositoryLiteDb.cs @@ -95,7 +95,7 @@ private async Task TryDeleteAllResourcesForGroupAsync(string resourceGroupId) var associatedBackupRecords = await _backupRecordPersistanceService.GetAllAsync(resourceGroupId); if (associatedBackupRecords != null) foreach (var record in associatedBackupRecords) - await _backupRecordPersistanceService.RemoveAsync(record.Id); + await _backupRecordPersistanceService.RemoveWithFileAsync(record.Id); } catch { } } From 34bbf983de9d5eef6f1f9397e8d7ea2bc7decf8a Mon Sep 17 00:00:00 2001 From: "George Njeri (Swagfin)" Date: Sat, 28 Jun 2025 00:10:11 +0300 Subject: [PATCH 3/6] chore: Additional refactor --- .../BackgroundJobs/BackupBackgroundZIPJob.cs | 29 ++++++---- .../BackgroundJobs/Bots/BackupZippingBot.cs | 56 ++++++------------- .../Bots/InDepthDeleteDropboxBot.cs | 4 +- .../Bots/InDepthDeleteObjectStorageBot.cs | 4 +- .../BackgroundJobs/Bots/MySQLBackupBot.cs | 4 +- .../BackgroundJobs/Bots/SQLBackupBot.cs | 4 +- .../Bots/UploaderAzureStorageBot.cs | 18 ++---- 7 files changed, 48 insertions(+), 71 deletions(-) diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs index d700a36..c73fe98 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs @@ -15,17 +15,27 @@ namespace SemanticBackup.Infrastructure.BackgroundJobs public class BackupBackgroundZIPJob : IHostedService { private readonly ILogger _logger; - private readonly IServiceScopeFactory _serviceScopeFactory; private readonly BotsManagerBackgroundJob _botsManagerBackgroundJob; + private readonly IResourceGroupRepository _resourceGroupRepository; + private readonly IDatabaseInfoRepository _databaseInfoRepository; + private readonly IBackupRecordRepository _backupRecordRepository; + public BackupBackgroundZIPJob( ILogger logger, IServiceScopeFactory serviceScopeFactory, - BotsManagerBackgroundJob botsManagerBackgroundJob) + BotsManagerBackgroundJob botsManagerBackgroundJob, + + IResourceGroupRepository resourceGroupRepository, + IDatabaseInfoRepository databaseInfoRepository, + IBackupRecordRepository backupRecordRepository + ) { this._logger = logger; - this._serviceScopeFactory = serviceScopeFactory; this._botsManagerBackgroundJob = botsManagerBackgroundJob; + this._resourceGroupRepository = resourceGroupRepository; + this._databaseInfoRepository = databaseInfoRepository; + this._backupRecordRepository = backupRecordRepository; } public Task StartAsync(CancellationToken cancellationToken) @@ -49,21 +59,16 @@ private void SetupBackgroundService(CancellationToken cancellationToken) await Task.Delay(7000, cancellationToken); try { - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - //DI INJECTIONS - IBackupRecordRepository backupRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); - IResourceGroupRepository resourceGroupPersistanceService = scope.ServiceProvider.GetRequiredService(); - IDatabaseInfoRepository databaseInfoRepository = scope.ServiceProvider.GetRequiredService(); //Proceed - List queuedBackups = await backupRecordPersistanceService.GetAllByStatusAsync(BackupRecordStatus.COMPLETED.ToString()); + List queuedBackups = await _backupRecordRepository.GetAllByStatusAsync(BackupRecordStatus.COMPLETED.ToString()); if (queuedBackups != null && queuedBackups.Count > 0) { foreach (BackupRecord backupRecord in queuedBackups.OrderBy(x => x.RegisteredDateUTC).ToList()) { //get valid database - BackupDatabaseInfo backupRecordDbInfo = await databaseInfoRepository.GetByIdAsync(backupRecord.BackupDatabaseInfoId); + BackupDatabaseInfo backupRecordDbInfo = await _databaseInfoRepository.GetByIdAsync(backupRecord.BackupDatabaseInfoId); //Check if valid Resource Group - ResourceGroup resourceGroup = await resourceGroupPersistanceService.GetByIdOrKeyAsync(backupRecordDbInfo?.ResourceGroupId ?? string.Empty); + ResourceGroup resourceGroup = await _resourceGroupRepository.GetByIdOrKeyAsync(backupRecordDbInfo?.ResourceGroupId ?? string.Empty); if (resourceGroup != null) { //Use Resource Group Threads @@ -74,7 +79,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) { _logger.LogInformation($"Queueing Zip Database Record Key: #{backupRecord.Id}..."); //Add to Queue - _botsManagerBackgroundJob.AddBot(new BackupZippingRobot(resourceGroup.Id, backupRecord, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new BackupZippingBot(resourceGroup.Id, backupRecord, _serviceScopeFactory)); bool updated = await backupRecordPersistanceService.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.COMPRESSING.ToString()); if (updated) _logger.LogInformation($"Queueing Zip Database Record Key: #{backupRecord.Id}...SUCCESS"); diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/BackupZippingBot.cs b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/BackupZippingBot.cs index 06da2a3..2d3033f 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/BackupZippingBot.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/BackupZippingBot.cs @@ -1,6 +1,5 @@ using ICSharpCode.SharpZipLib.Zip; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; +using SemanticBackup.Core.Helpers; using SemanticBackup.Core.Models; using System; using System.Diagnostics; @@ -10,24 +9,19 @@ namespace SemanticBackup.Infrastructure.BackgroundJobs.Bots { - internal class BackupZippingRobot : IBot + internal class BackupZippingBot : IBot { private readonly string _resourceGroupId; private readonly BackupRecord _backupRecord; - private readonly IServiceScopeFactory _scopeFactory; - private readonly ILogger _logger; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; - public string BotId => $"{_resourceGroupId}::{_backupRecord.Id}::{nameof(BackupZippingRobot)}"; + public string BotId => $"{_resourceGroupId}::{_backupRecord.Id}::{nameof(BackupZippingBot)}"; public string ResourceGroupId => _resourceGroupId; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public BackupZippingRobot(string resourceGroupId, BackupRecord backupRecord, IServiceScopeFactory scopeFactory) + public BackupZippingBot(string resourceGroupId, BackupRecord backupRecord) { _resourceGroupId = resourceGroupId; _backupRecord = backupRecord; - _scopeFactory = scopeFactory; - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); } public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) @@ -36,7 +30,7 @@ public async Task RunAsync(Func { - try - { - attempts++; - if (File.Exists(path)) - File.Delete(path); - success = true; - } - catch (Exception ex) - { - if (attempts >= 10) - { - Thread.Sleep(2000); - throw new Exception($"Maximum Deletion Attempts, Error: {ex.Message}"); - } - } - } - while (!success); - } - catch (Exception ex) - { - this._logger.LogWarning("The File Name Failed to Delete, Error: {Message}, File: {Path}", ex.Message, path); + //check file exists + if (File.Exists(path)) + File.Delete(path); + return Task.CompletedTask; + + }, 3, TimeSpan.FromSeconds(5)); } + catch (Exception ex) { Console.WriteLine($"Failed to remove File after compression: {ex.Message}"); } } } } diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/InDepthDeleteDropboxBot.cs b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/InDepthDeleteDropboxBot.cs index a509fb6..a5a31f2 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/InDepthDeleteDropboxBot.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/InDepthDeleteDropboxBot.cs @@ -31,7 +31,7 @@ public async Task RunAsync(Func _logger; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; public string BotId => $"{_resourceGroup.Id}::{_backupRecord.Id}::{nameof(UploaderAzureStorageBot)}"; public string ResourceGroupId => _resourceGroup.Id; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public UploaderAzureStorageBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord, IServiceScopeFactory scopeFactory) + public UploaderAzureStorageBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord) { _contentDeliveryRecord = contentDeliveryRecord; _resourceGroup = resourceGroup; _backupRecord = backupRecord; - _scopeFactory = scopeFactory; - //Logger - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); } + public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) { Status = BotStatus.Starting; Stopwatch stopwatch = new(); try { - _logger.LogInformation("uploading file to AzureBlobStorage: {Path}", _backupRecord.Path); + Console.WriteLine($"uploading file to AzureBlobStorage: {_backupRecord.Path}"); //proceed await Task.Delay(Random.Shared.Next(1000), cancellationToken); AzureBlobStorageDeliveryConfig settings = _resourceGroup.BackupDeliveryConfig.AzureBlobStorage ?? throw new Exception("no valid azure blob storage config"); @@ -75,13 +68,14 @@ await onDeliveryFeedUpdate(new BackupRecordDeliveryFeed Message = executionMessage, ElapsedMilliseconds = stopwatch.ElapsedMilliseconds }, cancellationToken); - _logger.LogInformation("Successfully uploaded file to AzureBlobStorage: {Path}", _backupRecord.Path); + + Console.WriteLine($"Successfully uploaded file to AzureBlobStorage: {_backupRecord.Path}"); Status = BotStatus.Completed; } catch (Exception ex) { + Console.WriteLine(ex.Message); Status = BotStatus.Error; - _logger.LogError(ex.Message); stopwatch.Stop(); await onDeliveryFeedUpdate(new BackupRecordDeliveryFeed { From e490a11917c61305fd6dfd3cd2152b5b236deedd Mon Sep 17 00:00:00 2001 From: "George Njeri (Swagfin)" Date: Sat, 28 Jun 2025 00:33:29 +0300 Subject: [PATCH 4/6] chore: Additional refactoring --- .../BackgroundJobs/BackupBackgroundZIPJob.cs | 4 +-- ...ckupRecordDeliveryDispatchBackgroundJob.cs | 5 +-- ...kupRecordDeliverySchedulerBackgroundJob.cs | 4 +-- .../BackupSchedulerBackgroundJob.cs | 3 -- .../BackgroundJobs/Bots/UploaderDropboxBot.cs | 31 +++-------------- .../Bots/UploaderEmailSMTPBot.cs | 19 ++++------- .../BackgroundJobs/Bots/UploaderFTPBot.cs | 34 +++++++------------ .../BackgroundJobs/Bots/UploaderLinkGenBot.cs | 21 ++++-------- .../Bots/UploaderObjectStorageBot.cs | 20 ++++------- .../BotsManagerBackgroundJob.cs | 19 +++++------ 10 files changed, 47 insertions(+), 113 deletions(-) diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs index c73fe98..5276c5e 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs @@ -1,5 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using SemanticBackup.Core.Interfaces; using SemanticBackup.Core.Models; @@ -23,7 +22,6 @@ public class BackupBackgroundZIPJob : IHostedService public BackupBackgroundZIPJob( ILogger logger, - IServiceScopeFactory serviceScopeFactory, BotsManagerBackgroundJob botsManagerBackgroundJob, IResourceGroupRepository resourceGroupRepository, diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs index e418e2d..bbda0c6 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs @@ -15,13 +15,11 @@ namespace SemanticBackup.Infrastructure.BackgroundJobs public class BackupRecordDeliveryDispatchBackgroundJob : IHostedService { private readonly ILogger _logger; - private readonly IServiceScopeFactory _serviceScopeFactory; private readonly BotsManagerBackgroundJob _botsManagerBackgroundJob; - public BackupRecordDeliveryDispatchBackgroundJob(ILogger logger, IServiceScopeFactory serviceScopeFactory, BotsManagerBackgroundJob botsManagerBackgroundJob) + public BackupRecordDeliveryDispatchBackgroundJob(ILogger logger, BotsManagerBackgroundJob botsManagerBackgroundJob) { _logger = logger; - _serviceScopeFactory = serviceScopeFactory; _botsManagerBackgroundJob = botsManagerBackgroundJob; } @@ -29,7 +27,6 @@ public Task StartAsync(CancellationToken cancellationToken) { _logger.LogInformation("Starting service...."); SetupBackgroundService(cancellationToken); - SetupBackgroundRemovedExpiredBackupsService(cancellationToken); return Task.CompletedTask; } diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs index 5b11b34..611dc64 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs @@ -15,11 +15,9 @@ namespace SemanticBackup.Infrastructure.BackgroundJobs public class BackupRecordDeliverySchedulerBackgroundJob : IHostedService { private readonly ILogger _logger; - private readonly IServiceScopeFactory _serviceScopeFactory; - public BackupRecordDeliverySchedulerBackgroundJob(ILogger logger, IServiceScopeFactory serviceScopeFactory) + public BackupRecordDeliverySchedulerBackgroundJob(ILogger logger) { this._logger = logger; - this._serviceScopeFactory = serviceScopeFactory; } public Task StartAsync(CancellationToken cancellationToken) diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs index 5b8a2a3..65bc1e3 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs @@ -18,17 +18,14 @@ public class BackupSchedulerBackgroundJob : IHostedService private readonly ILogger _logger; private readonly SystemConfigOptions _persistanceOptions; private readonly BotsManagerBackgroundJob _botsManagerBackgroundJob; - private readonly IServiceScopeFactory _serviceScopeFactory; public BackupSchedulerBackgroundJob( ILogger logger, SystemConfigOptions persistanceOptions, - IServiceScopeFactory serviceScopeFactory, BotsManagerBackgroundJob botsManagerBackgroundJob) { this._logger = logger; this._persistanceOptions = persistanceOptions; - this._serviceScopeFactory = serviceScopeFactory; this._botsManagerBackgroundJob = botsManagerBackgroundJob; } public Task StartAsync(CancellationToken cancellationToken) diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/UploaderDropboxBot.cs b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/UploaderDropboxBot.cs index b944477..e32a5a2 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/UploaderDropboxBot.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/UploaderDropboxBot.cs @@ -1,8 +1,5 @@ using Dropbox.Api; using Dropbox.Api.Files; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using SemanticBackup.Core.Interfaces; using SemanticBackup.Core.Models; using System; using System.Diagnostics; @@ -17,22 +14,16 @@ internal class UploaderDropboxBot : IBot private readonly BackupRecordDelivery _contentDeliveryRecord; private readonly ResourceGroup _resourceGroup; private readonly BackupRecord _backupRecord; - private readonly IServiceScopeFactory _scopeFactory; - private readonly ILogger _logger; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; public string BotId => $"{_resourceGroup.Id}::{_backupRecord.Id}::{nameof(UploaderDropboxBot)}"; public string ResourceGroupId => _resourceGroup.Id; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public UploaderDropboxBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord, IServiceScopeFactory scopeFactory) + public UploaderDropboxBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord) { _contentDeliveryRecord = contentDeliveryRecord; _resourceGroup = resourceGroup; _backupRecord = backupRecord; - _scopeFactory = scopeFactory; - //Logger - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); } public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) @@ -41,7 +32,7 @@ public async Task RunAsync(Func(); - _persistanceService.UpdateStatusFeedAsync(recordId, status, message, elapsed); - } - catch (Exception ex) - { - _logger.LogError("Error Updating Feed: {Message}", ex.Message); - } - } } } diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/UploaderEmailSMTPBot.cs b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/UploaderEmailSMTPBot.cs index 6a07506..2cc8bd7 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/UploaderEmailSMTPBot.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/UploaderEmailSMTPBot.cs @@ -1,6 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using SemanticBackup.Core; +using SemanticBackup.Core; using SemanticBackup.Core.Models; using System; using System.Collections.Generic; @@ -18,22 +16,16 @@ internal class UploaderEmailSMTPBot : IBot private readonly BackupRecordDelivery _contentDeliveryRecord; private readonly ResourceGroup _resourceGroup; private readonly BackupRecord _backupRecord; - private readonly IServiceScopeFactory _scopeFactory; - private readonly ILogger _logger; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; public string BotId => $"{_resourceGroup.Id}::{_backupRecord.Id}::{nameof(UploaderEmailSMTPBot)}"; public string ResourceGroupId => _resourceGroup.Id; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public UploaderEmailSMTPBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord, IServiceScopeFactory scopeFactory) + public UploaderEmailSMTPBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord) { _contentDeliveryRecord = contentDeliveryRecord; _resourceGroup = resourceGroup; _backupRecord = backupRecord; - _scopeFactory = scopeFactory; - //Logger - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); } public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) @@ -42,7 +34,7 @@ public async Task RunAsync(Func _logger; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; public string BotId => $"{_resourceGroup.Id}::{_backupRecord.Id}::{nameof(UploaderFTPBot)}"; public string ResourceGroupId => _resourceGroup.Id; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public UploaderFTPBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord, IServiceScopeFactory scopeFactory) + public UploaderFTPBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord) { _contentDeliveryRecord = contentDeliveryRecord; _resourceGroup = resourceGroup; _backupRecord = backupRecord; - _scopeFactory = scopeFactory; - //Logger - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); } public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) @@ -39,7 +31,7 @@ public async Task RunAsync(Func _logger; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; public string BotId => $"{_resourceGroup.Id}::{_backupRecord.Id}::{nameof(UploaderLinkGenBot)}"; public string ResourceGroupId => _resourceGroup.Id; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public UploaderLinkGenBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord, IServiceScopeFactory scopeFactory) + public UploaderLinkGenBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord) { _contentDeliveryRecord = contentDeliveryRecord; _resourceGroup = resourceGroup; _backupRecord = backupRecord; - _scopeFactory = scopeFactory; - //Logger - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); } public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) @@ -39,14 +31,14 @@ public async Task RunAsync(Func _logger; public DateTime DateCreatedUtc { get; set; } = DateTime.UtcNow; public string BotId => $"{_resourceGroup.Id}::{_backupRecord.Id}::{nameof(UploaderObjectStorageBot)}"; public string ResourceGroupId => _resourceGroup.Id; public BotStatus Status { get; internal set; } = BotStatus.NotReady; - public UploaderObjectStorageBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord, IServiceScopeFactory scopeFactory) + public UploaderObjectStorageBot(ResourceGroup resourceGroup, BackupRecord backupRecord, BackupRecordDelivery contentDeliveryRecord) { _contentDeliveryRecord = contentDeliveryRecord; _resourceGroup = resourceGroup; _backupRecord = backupRecord; - _scopeFactory = scopeFactory; - //Logger - using IServiceScope scope = _scopeFactory.CreateScope(); - _logger = scope.ServiceProvider.GetRequiredService>(); } + public async Task RunAsync(Func onDeliveryFeedUpdate, CancellationToken cancellationToken) { Status = BotStatus.Starting; Stopwatch stopwatch = new(); try { - _logger.LogInformation("uploading file to ObjectStorage: {Path}", _backupRecord.Path); + Console.WriteLine($"uploading file to ObjectStorage: {_backupRecord.Path}"); //proceed await Task.Delay(Random.Shared.Next(1000), cancellationToken); ObjectStorageDeliveryConfig settings = _resourceGroup.BackupDeliveryConfig.ObjectStorage ?? throw new Exception("no valid object storage config"); @@ -85,13 +78,14 @@ await onDeliveryFeedUpdate(new BackupRecordDeliveryFeed Message = executionMessage, ElapsedMilliseconds = stopwatch.ElapsedMilliseconds }, cancellationToken); - _logger.LogInformation("Successfully uploaded file to ObjectStorage: {Path}", _backupRecord.Path); + + Console.WriteLine($"Successfully uploaded file to ObjectStorage: {_backupRecord.Path}"); Status = BotStatus.Completed; } catch (Exception ex) { Status = BotStatus.Error; - _logger.LogError(ex.Message); + Console.WriteLine(ex.Message); stopwatch.Stop(); await onDeliveryFeedUpdate(new BackupRecordDeliveryFeed { diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BotsManagerBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BotsManagerBackgroundJob.cs index 83176d9..7962ee3 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BotsManagerBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BotsManagerBackgroundJob.cs @@ -1,5 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using SemanticBackup.Core.Interfaces; using SemanticBackup.Infrastructure.BackgroundJobs.Bots; @@ -14,14 +13,16 @@ namespace SemanticBackup.Infrastructure.BackgroundJobs public class BotsManagerBackgroundJob : IHostedService { private readonly ILogger _logger; - private readonly IServiceScopeFactory _serviceScopeFactory; + private readonly IBackupRecordRepository _backupRecordRepository; + private readonly IContentDeliveryRecordRepository _deliveryRecordRepository; private List Bots { get; set; } = []; - public BotsManagerBackgroundJob(ILogger logger, IServiceScopeFactory serviceScopeFactory) + public BotsManagerBackgroundJob(ILogger logger, IBackupRecordRepository backupRecordRepository, IContentDeliveryRecordRepository deliveryRecordRepository) { _logger = logger; - _serviceScopeFactory = serviceScopeFactory; + _backupRecordRepository = backupRecordRepository; + _deliveryRecordRepository = deliveryRecordRepository; } public Task StartAsync(CancellationToken cancellationToken) @@ -91,15 +92,11 @@ private async Task OnDeliveryFeedUpdate(BackupRecordDeliveryFeed feed, Cancellat { if (feed.DeliveryFeedType == DeliveryFeedType.BackupNotify && feed.BackupRecordId > 0) { - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - IBackupRecordRepository _backupRecordRepo = scope.ServiceProvider.GetRequiredService(); - await _backupRecordRepo.UpdateStatusFeedAsync(feed.BackupRecordId, feed.Status.ToString(), feed.Message, feed.ElapsedMilliseconds, feed.NewFilePath); + await _backupRecordRepository.UpdateStatusFeedAsync(feed.BackupRecordId, feed.Status.ToString(), feed.Message, feed.ElapsedMilliseconds, feed.NewFilePath); } else if (feed.DeliveryFeedType == DeliveryFeedType.BackupDeliveryNotify && !string.IsNullOrWhiteSpace(feed.BackupRecordDeliveryId)) { - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - IContentDeliveryRecordRepository _contentDeliveryRepo = scope.ServiceProvider.GetRequiredService(); - await _contentDeliveryRepo.UpdateStatusFeedAsync(feed.BackupRecordDeliveryId, feed.Status.ToString(), feed.Message, feed.ElapsedMilliseconds); + await _deliveryRecordRepository.UpdateStatusFeedAsync(feed.BackupRecordDeliveryId, feed.Status.ToString(), feed.Message, feed.ElapsedMilliseconds); } else throw new Exception($"unsupported delivery-feed-type: {feed.DeliveryFeedType}"); From 296c9046d30cfd850d28ab484892fe042211b5e0 Mon Sep 17 00:00:00 2001 From: "George Njeri (Swagfin)" Date: Sat, 28 Jun 2025 00:50:16 +0300 Subject: [PATCH 5/6] chore: additional refactoring on bots --- .../BackgroundJobs/BackupBackgroundZIPJob.cs | 6 +- ...ckupRecordDeliveryDispatchBackgroundJob.cs | 54 +++++++----- ...kupRecordDeliverySchedulerBackgroundJob.cs | 43 ++++++---- .../BackupSchedulerBackgroundJob.cs | 82 ++++++++++--------- .../BackupProviderForMySQLServer.cs | 60 ++++++-------- .../BackupProviderForSQLServer.cs | 76 ++++++++--------- .../BackupRecordRepositoryLiteDb.cs | 18 ++-- .../ContentDeliveryRecordRepositoryLiteDb.cs | 6 +- .../DatabaseInfoRepositoryLiteDb.cs | 4 +- .../ResourceGroupRepositoryLiteDb.cs | 10 +-- 10 files changed, 183 insertions(+), 176 deletions(-) diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs index 5276c5e..6a75e7b 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs @@ -77,8 +77,8 @@ private void SetupBackgroundService(CancellationToken cancellationToken) { _logger.LogInformation($"Queueing Zip Database Record Key: #{backupRecord.Id}..."); //Add to Queue - _botsManagerBackgroundJob.AddBot(new BackupZippingBot(resourceGroup.Id, backupRecord, _serviceScopeFactory)); - bool updated = await backupRecordPersistanceService.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.COMPRESSING.ToString()); + _botsManagerBackgroundJob.AddBot(new BackupZippingBot(resourceGroup.Id, backupRecord)); + bool updated = await _backupRecordRepository.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.COMPRESSING.ToString()); if (updated) _logger.LogInformation($"Queueing Zip Database Record Key: #{backupRecord.Id}...SUCCESS"); else @@ -88,7 +88,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) else { _logger.LogInformation($">> Skipping Compression for Database Record Key: #{backupRecord.Id}..."); - bool updated = await backupRecordPersistanceService.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.READY.ToString()); + bool updated = await _backupRecordRepository.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.READY.ToString()); if (updated) _logger.LogInformation($">> Skipped Compression and Completed Backup Updated Record Key: #{backupRecord.Id}...SUCCESS"); else diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs index bbda0c6..5c5ecb3 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs @@ -1,5 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using SemanticBackup.Core.Interfaces; using SemanticBackup.Core.Models; @@ -17,10 +16,27 @@ public class BackupRecordDeliveryDispatchBackgroundJob : IHostedService private readonly ILogger _logger; private readonly BotsManagerBackgroundJob _botsManagerBackgroundJob; - public BackupRecordDeliveryDispatchBackgroundJob(ILogger logger, BotsManagerBackgroundJob botsManagerBackgroundJob) + private readonly IResourceGroupRepository _resourceGroupRepository; + private readonly IBackupRecordRepository _backupRecordRepository; + private readonly IContentDeliveryRecordRepository _deliveryRecordRepository; + private readonly IDatabaseInfoRepository _databaseInfoRepository; + + public BackupRecordDeliveryDispatchBackgroundJob( + ILogger logger, + BotsManagerBackgroundJob botsManagerBackgroundJob, + + IResourceGroupRepository resourceGroupRepository, + IBackupRecordRepository backupRecordRepository, + IContentDeliveryRecordRepository contentDeliveryRecordRepository, + IDatabaseInfoRepository databaseInfoRepository + ) { _logger = logger; _botsManagerBackgroundJob = botsManagerBackgroundJob; + _resourceGroupRepository = resourceGroupRepository; + _backupRecordRepository = backupRecordRepository; + _deliveryRecordRepository = contentDeliveryRecordRepository; + _databaseInfoRepository = databaseInfoRepository; } public Task StartAsync(CancellationToken cancellationToken) @@ -44,23 +60,17 @@ private void SetupBackgroundService(CancellationToken cancellationToken) await Task.Delay(10000, cancellationToken); try { - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - //DI INJECTIONS - IContentDeliveryRecordRepository contentDeliveryRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); - IBackupRecordRepository backupRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); - IResourceGroupRepository resourceGroupPersistanceService = scope.ServiceProvider.GetRequiredService(); - IDatabaseInfoRepository databaseInfoPersistanceService = scope.ServiceProvider.GetRequiredService(); //Proceed - List contentDeliveryRecords = await contentDeliveryRecordPersistanceService.GetAllByStatusAsync(BackupRecordDeliveryStatus.QUEUED.ToString()); + List contentDeliveryRecords = await _deliveryRecordRepository.GetAllByStatusAsync(BackupRecordDeliveryStatus.QUEUED.ToString()); if (contentDeliveryRecords != null && contentDeliveryRecords.Count > 0) { List scheduleToDeleteRecords = []; foreach (BackupRecordDelivery contentDeliveryRecord in contentDeliveryRecords.OrderBy(x => x.RegisteredDateUTC).ToList()) { _logger.LogInformation($"Processing Queued Content Delivery Record: #{contentDeliveryRecord.Id}..."); - BackupRecord backupRecordInfo = await backupRecordPersistanceService.GetByIdAsync(contentDeliveryRecord?.BackupRecordId ?? 0); - BackupDatabaseInfo backupDatabaseInfo = await databaseInfoPersistanceService.GetByIdAsync(backupRecordInfo?.BackupDatabaseInfoId); - ResourceGroup resourceGroup = await resourceGroupPersistanceService.GetByIdOrKeyAsync(backupDatabaseInfo?.ResourceGroupId); + BackupRecord backupRecordInfo = await _backupRecordRepository.GetByIdAsync(contentDeliveryRecord?.BackupRecordId ?? 0); + BackupDatabaseInfo backupDatabaseInfo = await _databaseInfoRepository.GetByIdAsync(backupRecordInfo?.BackupDatabaseInfoId); + ResourceGroup resourceGroup = await _resourceGroupRepository.GetByIdOrKeyAsync(backupDatabaseInfo?.ResourceGroupId); if (backupRecordInfo == null) { @@ -87,32 +97,32 @@ private void SetupBackgroundService(CancellationToken cancellationToken) if (contentDeliveryRecord.DeliveryType == BackupDeliveryConfigTypes.DownloadLink.ToString()) { //Download Link Generator - _botsManagerBackgroundJob.AddBot(new UploaderLinkGenBot(resourceGroup, backupRecordInfo, contentDeliveryRecord, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new UploaderLinkGenBot(resourceGroup, backupRecordInfo, contentDeliveryRecord)); } else if (contentDeliveryRecord.DeliveryType == BackupDeliveryConfigTypes.Ftp.ToString()) { //FTP Uploader - _botsManagerBackgroundJob.AddBot(new UploaderFTPBot(resourceGroup, backupRecordInfo, contentDeliveryRecord, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new UploaderFTPBot(resourceGroup, backupRecordInfo, contentDeliveryRecord)); } else if (contentDeliveryRecord.DeliveryType == BackupDeliveryConfigTypes.Smtp.ToString()) { //Email Send and Uploader - _botsManagerBackgroundJob.AddBot(new UploaderEmailSMTPBot(resourceGroup, backupRecordInfo, contentDeliveryRecord, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new UploaderEmailSMTPBot(resourceGroup, backupRecordInfo, contentDeliveryRecord)); } else if (contentDeliveryRecord.DeliveryType == BackupDeliveryConfigTypes.Dropbox.ToString()) { //Email Send and Uploader - _botsManagerBackgroundJob.AddBot(new UploaderDropboxBot(resourceGroup, backupRecordInfo, contentDeliveryRecord, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new UploaderDropboxBot(resourceGroup, backupRecordInfo, contentDeliveryRecord)); } else if (contentDeliveryRecord.DeliveryType == BackupDeliveryConfigTypes.AzureBlobStorage.ToString()) { //Azure Blob Storage - _botsManagerBackgroundJob.AddBot(new UploaderAzureStorageBot(resourceGroup, backupRecordInfo, contentDeliveryRecord, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new UploaderAzureStorageBot(resourceGroup, backupRecordInfo, contentDeliveryRecord)); } else if (contentDeliveryRecord.DeliveryType == BackupDeliveryConfigTypes.ObjectStorage.ToString()) { //Object Storage - _botsManagerBackgroundJob.AddBot(new UploaderObjectStorageBot(resourceGroup, backupRecordInfo, contentDeliveryRecord, _serviceScopeFactory)); + _botsManagerBackgroundJob.AddBot(new UploaderObjectStorageBot(resourceGroup, backupRecordInfo, contentDeliveryRecord)); } else { @@ -122,7 +132,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) scheduleToDeleteRecords.Add(contentDeliveryRecord.Id); } //Finally Update Status - bool updated = await contentDeliveryRecordPersistanceService.UpdateStatusFeedAsync(contentDeliveryRecord.Id, status, statusMsg); + bool updated = await _deliveryRecordRepository.UpdateStatusFeedAsync(contentDeliveryRecord.Id, status, statusMsg); if (!updated) _logger.LogWarning($"Queued for Backup but was unable to update backup record Key: #{contentDeliveryRecord.Id} status"); } @@ -131,8 +141,8 @@ private void SetupBackgroundService(CancellationToken cancellationToken) } } //Check if Any Delete - foreach (string rm in scheduleToDeleteRecords) - await contentDeliveryRecordPersistanceService.RemoveAsync(rm); + foreach (string contentDeliveryRecordId in scheduleToDeleteRecords) + await _deliveryRecordRepository.RemoveAsync(contentDeliveryRecordId); } } catch (Exception ex) diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs index 611dc64..4d32e17 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs @@ -1,5 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using SemanticBackup.Core; using SemanticBackup.Core.Interfaces; @@ -15,9 +14,26 @@ namespace SemanticBackup.Infrastructure.BackgroundJobs public class BackupRecordDeliverySchedulerBackgroundJob : IHostedService { private readonly ILogger _logger; - public BackupRecordDeliverySchedulerBackgroundJob(ILogger logger) + + private readonly IResourceGroupRepository _resourceGroupRepository; + private readonly IBackupRecordRepository _backupRecordRepository; + private readonly IContentDeliveryRecordRepository _deliveryRecordRepository; + private readonly IDatabaseInfoRepository _databaseInfoRepository; + + public BackupRecordDeliverySchedulerBackgroundJob( + ILogger logger, + + IResourceGroupRepository resourceGroupRepository, + IBackupRecordRepository backupRecordRepository, + IContentDeliveryRecordRepository contentDeliveryRecordRepository, + IDatabaseInfoRepository databaseInfoRepository + ) { - this._logger = logger; + _logger = logger; + _resourceGroupRepository = resourceGroupRepository; + _backupRecordRepository = backupRecordRepository; + _deliveryRecordRepository = contentDeliveryRecordRepository; + _databaseInfoRepository = databaseInfoRepository; } public Task StartAsync(CancellationToken cancellationToken) @@ -41,29 +57,22 @@ private void SetupBackgroundService(CancellationToken cancellationToken) await Task.Delay(4000, cancellationToken); try { - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - //DI INJECTIONS - IBackupRecordRepository backupRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); //Proceed - List pendingExecutionRecords = await backupRecordPersistanceService.GetAllReadyAndPendingDeliveryAsync(); + List pendingExecutionRecords = await _backupRecordRepository.GetAllReadyAndPendingDeliveryAsync(); foreach (BackupRecord backupRecord in pendingExecutionRecords?.OrderBy(x => x.RegisteredDateUTC)?.ToList()) { _logger.LogInformation($"Queueing Content Delivery for Backup Record Id: {backupRecord.Id}..."); - //## get other services - IResourceGroupRepository resourceGroupPersistanceService = scope.ServiceProvider.GetRequiredService(); - IContentDeliveryRecordRepository contentDeliveryRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); - IDatabaseInfoRepository databaseInfoRepository = scope.ServiceProvider.GetRequiredService(); //get db information - BackupDatabaseInfo backupRecordDbInfo = await databaseInfoRepository.GetByIdAsync(backupRecord.BackupDatabaseInfoId); + BackupDatabaseInfo backupRecordDbInfo = await _databaseInfoRepository.GetByIdAsync(backupRecord.BackupDatabaseInfoId); //Check if valid Resource Group - ResourceGroup resourceGroup = await resourceGroupPersistanceService.GetByIdOrKeyAsync(backupRecordDbInfo?.ResourceGroupId ?? string.Empty); + ResourceGroup resourceGroup = await _resourceGroupRepository.GetByIdOrKeyAsync(backupRecordDbInfo?.ResourceGroupId ?? string.Empty); //Has Valid Resource Group //check if backup delivery config is set if (resourceGroup.BackupDeliveryConfig == null) { _logger.LogInformation($"Resource Group Id: {backupRecord.Id}, doesn't have any backup delivery config, Skipped"); - _ = await backupRecordPersistanceService.UpdateDeliveryRunnedAsync(backupRecord.Id, true, BackupRecordExecutedDeliveryRunStatus.SKIPPED_EXECUTION.ToString()); + _ = await _backupRecordRepository.UpdateDeliveryRunnedAsync(backupRecord.Id, true, BackupRecordExecutedDeliveryRunStatus.SKIPPED_EXECUTION.ToString()); } else { @@ -84,7 +93,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) //check if enabled if (isDeliveryEnabled) { - bool queuedSuccess = await contentDeliveryRecordPersistanceService.AddOrUpdateAsync(new BackupRecordDelivery + bool queuedSuccess = await _deliveryRecordRepository.AddOrUpdateAsync(new BackupRecordDelivery { Id = $"{backupRecord.Id}|{resourceGroup.Id}|{deliveryType}".ToMD5String().ToUpper(), //Unique Identification BackupRecordId = backupRecord.Id, @@ -99,7 +108,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) } } //Update Execution - _ = await backupRecordPersistanceService.UpdateDeliveryRunnedAsync(backupRecord.Id, true, BackupRecordExecutedDeliveryRunStatus.SUCCESSFULLY_EXECUTED.ToString()); + _ = await _backupRecordRepository.UpdateDeliveryRunnedAsync(backupRecord.Id, true, BackupRecordExecutedDeliveryRunStatus.SUCCESSFULLY_EXECUTED.ToString()); } } } diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs index 65bc1e3..c9a2c46 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs @@ -1,5 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using SemanticBackup.Core; using SemanticBackup.Core.Interfaces; @@ -19,15 +18,35 @@ public class BackupSchedulerBackgroundJob : IHostedService private readonly SystemConfigOptions _persistanceOptions; private readonly BotsManagerBackgroundJob _botsManagerBackgroundJob; + private readonly IResourceGroupRepository _resourceGroupRepository; + private readonly IBackupScheduleRepository _backupScheduleRepository; + private readonly IBackupRecordRepository _backupRecordRepository; + private readonly IContentDeliveryRecordRepository _deliveryRecordRepository; + private readonly IDatabaseInfoRepository _databaseInfoRepository; + public BackupSchedulerBackgroundJob( ILogger logger, SystemConfigOptions persistanceOptions, - BotsManagerBackgroundJob botsManagerBackgroundJob) + BotsManagerBackgroundJob botsManagerBackgroundJob, + + IResourceGroupRepository resourceGroupRepository, + IBackupScheduleRepository backupScheduleRepository, + IBackupRecordRepository backupRecordRepository, + IContentDeliveryRecordRepository contentDeliveryRecordRepository, + IDatabaseInfoRepository databaseInfoRepository + ) { - this._logger = logger; - this._persistanceOptions = persistanceOptions; - this._botsManagerBackgroundJob = botsManagerBackgroundJob; + _logger = logger; + _persistanceOptions = persistanceOptions; + _botsManagerBackgroundJob = botsManagerBackgroundJob; + + _resourceGroupRepository = resourceGroupRepository; + _backupScheduleRepository = backupScheduleRepository; + _backupRecordRepository = backupRecordRepository; + _deliveryRecordRepository = contentDeliveryRecordRepository; + _databaseInfoRepository = databaseInfoRepository; } + public Task StartAsync(CancellationToken cancellationToken) { _logger.LogInformation("Starting service...."); @@ -49,22 +68,16 @@ private void SetupBackgroundService(CancellationToken cancellationToken) await Task.Delay(3000, cancellationToken); try { - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - //DI INJECTIONS - IBackupScheduleRepository backupSchedulePersistanceService = scope.ServiceProvider.GetRequiredService(); - IDatabaseInfoRepository databaseInfoPersistanceService = scope.ServiceProvider.GetRequiredService(); - IResourceGroupRepository resourceGroupPersistanceService = scope.ServiceProvider.GetRequiredService(); - IBackupRecordRepository backupRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); //Proceed DateTime currentTimeUTC = DateTime.UtcNow; - List dueSchedules = await backupSchedulePersistanceService.GetAllDueByDateAsync(); + List dueSchedules = await _backupScheduleRepository.GetAllDueByDateAsync(); if (dueSchedules != null && dueSchedules.Count > 0) { List scheduleToDelete = []; foreach (BackupSchedule schedule in dueSchedules.OrderBy(x => x.NextRunUTC).ToList()) { _logger.LogInformation($"Queueing Scheduled Backup..."); - BackupDatabaseInfo backupDatabaseInfo = await databaseInfoPersistanceService.GetByIdAsync(schedule.BackupDatabaseInfoId); + BackupDatabaseInfo backupDatabaseInfo = await _databaseInfoRepository.GetByIdAsync(schedule.BackupDatabaseInfoId); if (backupDatabaseInfo == null) { _logger.LogWarning($"No Database Info matches with Id: {schedule.BackupDatabaseInfoId}, Schedule Record will be Deleted: {schedule.Id}"); @@ -73,7 +86,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) else { //Proceed - ResourceGroup resourceGroup = await resourceGroupPersistanceService.GetByIdOrKeyAsync(backupDatabaseInfo.ResourceGroupId); + ResourceGroup resourceGroup = await _resourceGroupRepository.GetByIdOrKeyAsync(backupDatabaseInfo.ResourceGroupId); if (resourceGroup == null) { _logger.LogWarning($"Can NOT queue Database for Backup Id: {backupDatabaseInfo.Id}, Reason: Assigned Resource Group doen't exist, Resource Group Id: {backupDatabaseInfo.Id}, Schedule will be Removed"); @@ -95,13 +108,13 @@ private void SetupBackgroundService(CancellationToken cancellationToken) ExecutedDeliveryRun = false }; - bool addedSuccess = await backupRecordPersistanceService.AddOrUpdateAsync(newRecord); + bool addedSuccess = await _backupRecordRepository.AddOrUpdateAsync(newRecord); if (!addedSuccess) throw new Exception("Unable to Queue Database for Backup"); else _logger.LogInformation($"Queueing Scheduled Backup...SUCCESS"); //Update Schedule - bool updatedSchedule = await backupSchedulePersistanceService.UpdateLastRunAsync(schedule.Id, currentTimeUTC); + bool updatedSchedule = await _backupScheduleRepository.UpdateLastRunAsync(schedule.Id, currentTimeUTC); if (!updatedSchedule) _logger.LogWarning("Unable to Update Scheduled Next Run"); //Buy Some Seconds to avoid Conflict Name @@ -112,9 +125,8 @@ private void SetupBackgroundService(CancellationToken cancellationToken) } //Check if Any Delete - if (scheduleToDelete.Count > 0) - foreach (var rm in scheduleToDelete) - await backupSchedulePersistanceService.RemoveAsync(rm); + foreach (string scheduleId in scheduleToDelete) + await _backupScheduleRepository.RemoveAsync(scheduleId); } } catch (Exception ex) @@ -136,29 +148,23 @@ private void SetupBackgroundNonResponsiveStopService(CancellationToken cancellat { try { - using IServiceScope scope = _serviceScopeFactory.CreateScope(); - //DI INJECTIONS - IBackupRecordRepository backupRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); - IContentDeliveryRecordRepository contentDeliveryRecordPersistanceService = scope.ServiceProvider.GetRequiredService(); //Proceed List botsToRemove = []; //REMOVE BACKUP RECORDS - List recordsIds = await backupRecordPersistanceService.GetAllNoneResponsiveIdsAsync(statusChecks, executionTimeoutInMinutes); - if (recordsIds != null && recordsIds.Count > 0) - foreach (long id in recordsIds) - { - await backupRecordPersistanceService.UpdateStatusFeedAsync(id, BackupRecordStatus.ERROR.ToString(), "Bot Execution Timeout", executionTimeoutInMinutes); - botsToRemove.Add(id.ToString()); - } + List recordsIds = (await _backupRecordRepository.GetAllNoneResponsiveIdsAsync(statusChecks, executionTimeoutInMinutes)) ?? []; + foreach (long id in recordsIds) + { + await _backupRecordRepository.UpdateStatusFeedAsync(id, BackupRecordStatus.ERROR.ToString(), "Bot Execution Timeout", executionTimeoutInMinutes); + botsToRemove.Add(id.ToString()); + } //REMOVE CONTENT DELIVERY RECORDS - List deliveryRecordIds = await contentDeliveryRecordPersistanceService.GetAllNoneResponsiveAsync(statusChecks, executionTimeoutInMinutes); - if (deliveryRecordIds != null && deliveryRecordIds.Count > 0) - foreach (string id in deliveryRecordIds) - { - await contentDeliveryRecordPersistanceService.UpdateStatusFeedAsync(id, BackupRecordStatus.ERROR.ToString(), "Bot Execution Timeout", executionTimeoutInMinutes); - botsToRemove.Add(id); - } + List deliveryRecordIds = (await _deliveryRecordRepository.GetAllNoneResponsiveAsync(statusChecks, executionTimeoutInMinutes)) ?? []; + foreach (string id in deliveryRecordIds) + { + await _deliveryRecordRepository.UpdateStatusFeedAsync(id, BackupRecordStatus.ERROR.ToString(), "Bot Execution Timeout", executionTimeoutInMinutes); + botsToRemove.Add(id); + } //Finally Try And Stop if (botsToRemove.Count > 0) diff --git a/SemanticBackup.Infrastructure/Implementations/BackupProviderForMySQLServer.cs b/SemanticBackup.Infrastructure/Implementations/BackupProviderForMySQLServer.cs index 79a4c8c..0b94b94 100644 --- a/SemanticBackup.Infrastructure/Implementations/BackupProviderForMySQLServer.cs +++ b/SemanticBackup.Infrastructure/Implementations/BackupProviderForMySQLServer.cs @@ -16,48 +16,40 @@ public async Task BackupDatabaseAsync(string databaseName, ResourceGroup r string connectionString = resourceGroup.GetDbConnectionString(databaseName); if (string.IsNullOrWhiteSpace(connectionString)) throw new Exception($"Invalid connection string provided for Database Type: {resourceGroup.DbType} is not Valid or is not Supported"); - using (MySqlConnection conn = new MySqlConnection(connectionString)) - { - using (MySqlCommand cmd = new MySqlCommand()) - { - using (MySqlBackup mb = new MySqlBackup(cmd)) - { - await conn.OpenAsync(); - cmd.Connection = conn; - mb.ExportToFile(backupRecord.Path.Trim()); - await conn.CloseAsync(); - } - } - return true; - } + using MySqlConnection conn = new MySqlConnection(connectionString); + using MySqlCommand cmd = new MySqlCommand(); + using MySqlBackup mb = new MySqlBackup(cmd); + await conn.OpenAsync(); + cmd.Connection = conn; + mb.ExportToFile(backupRecord.Path.Trim()); + await conn.CloseAsync(); + return true; } public async Task> GetAvailableDatabaseCollectionAsync(ResourceGroup resourceGroup) { - List availableDbs = new List(); - string[] exclude = new string[] { "information_schema", "mysql", "performance_schema" }; + List availableDbs = []; + string[] exclude = ["information_schema", "mysql", "performance_schema"]; string connectionString = resourceGroup.GetDbConnectionString(); using (MySqlConnection conn = new MySqlConnection(connectionString)) { - using (MySqlCommand cmd = new MySqlCommand("SHOW DATABASES;")) + using MySqlCommand cmd = new MySqlCommand("SHOW DATABASES;"); + await conn.OpenAsync(); + cmd.Connection = conn; + using (var reader = await cmd.ExecuteReaderAsync()) { - await conn.OpenAsync(); - cmd.Connection = conn; - using (var reader = await cmd.ExecuteReaderAsync()) + if (reader.HasRows) { - if (reader.HasRows) + while (await reader.ReadAsync()) { - while (await reader.ReadAsync()) - { - string dbName = reader?.GetString(0); - if (!exclude.Contains(dbName)) - availableDbs.Add(dbName); - } + string dbName = reader?.GetString(0); + if (!exclude.Contains(dbName)) + availableDbs.Add(dbName); } - await reader.CloseAsync(); } - await conn.CloseAsync(); + await reader.CloseAsync(); } + await conn.CloseAsync(); } return availableDbs; } @@ -67,12 +59,10 @@ public async Task> GetAvailableDatabaseCollectionAsync(ResourceGrou try { string connectionString = resourceGroup.GetDbConnectionString(); - using (MySqlConnection conn = new MySqlConnection(connectionString)) - { - await conn.OpenAsync(); - await conn.CloseAsync(); - return (true, string.Empty); - } + using MySqlConnection conn = new MySqlConnection(connectionString); + await conn.OpenAsync(); + await conn.CloseAsync(); + return (true, string.Empty); } catch (Exception ex) { diff --git a/SemanticBackup.Infrastructure/Implementations/BackupProviderForSQLServer.cs b/SemanticBackup.Infrastructure/Implementations/BackupProviderForSQLServer.cs index 8eedd41..7998d6f 100644 --- a/SemanticBackup.Infrastructure/Implementations/BackupProviderForSQLServer.cs +++ b/SemanticBackup.Infrastructure/Implementations/BackupProviderForSQLServer.cs @@ -22,17 +22,15 @@ BACKUP DATABASE [{0}] string connectionString = resourceGroup.GetDbConnectionString(); if (string.IsNullOrWhiteSpace(connectionString)) throw new Exception($"Invalid connection string provided for Database Type: {resourceGroup.DbType} is not Valid or is not Supported"); - using (DbConnection connection = new SqlConnection(connectionString)) - { - await connection.OpenAsync(); - DbCommand command = connection.CreateCommand(); - command.CommandTimeout = 0; // Backups can take a long time for big databases - command.CommandText = string.Format(backupCommandTemplate, databaseName, backupRecord.Path.Trim()); - //Execute - int queryRows = await command.ExecuteNonQueryAsync(); - await connection.CloseAsync(); - return true; - } + using DbConnection connection = new SqlConnection(connectionString); + await connection.OpenAsync(); + DbCommand command = connection.CreateCommand(); + command.CommandTimeout = 0; // Backups can take a long time for big databases + command.CommandText = string.Format(backupCommandTemplate, databaseName, backupRecord.Path.Trim()); + //Execute + int queryRows = await command.ExecuteNonQueryAsync(); + await connection.CloseAsync(); + return true; } public async Task RestoreDatabaseAsync(string databaseName, ResourceGroup resourceGroup, BackupRecord backupRecord) @@ -48,45 +46,41 @@ RESTORE DATABASE [{0}] throw new Exception($"Invalid connection string provided for Database Type: {resourceGroup.DbType} is not Valid or is not Supported"); if (string.IsNullOrEmpty(backupRecord.Path)) throw new Exception("Source Location can't be NULL"); - using (DbConnection connection = new SqlConnection(connectionString)) - { - await connection.OpenAsync(); - DbCommand command = connection.CreateCommand(); - command.CommandTimeout = 0; // Backups can take a long time for big databases - command.CommandText = string.Format(restoreCommandTemplate, databaseName, backupRecord.Path); - //Execute - int queryRows = await command.ExecuteNonQueryAsync(); - connection.Close(); - return true; - } + using DbConnection connection = new SqlConnection(connectionString); + await connection.OpenAsync(); + DbCommand command = connection.CreateCommand(); + command.CommandTimeout = 0; // Backups can take a long time for big databases + command.CommandText = string.Format(restoreCommandTemplate, databaseName, backupRecord.Path); + //Execute + int queryRows = await command.ExecuteNonQueryAsync(); + connection.Close(); + return true; } public async Task> GetAvailableDatabaseCollectionAsync(ResourceGroup resourceGroup) { List availableDbs = new List(); - string[] exclude = new string[] { "master", "model", "msdb", "tempdb" }; + string[] exclude = ["master", "model", "msdb", "tempdb"]; string connectionString = resourceGroup.GetDbConnectionString(); using (SqlConnection conn = new SqlConnection(connectionString)) { - using (SqlCommand cmd = new SqlCommand("SELECT name FROM master.dbo.sysdatabases")) + using SqlCommand cmd = new SqlCommand("SELECT name FROM master.dbo.sysdatabases"); + await conn.OpenAsync(); + cmd.Connection = conn; + using (var reader = await cmd.ExecuteReaderAsync()) { - await conn.OpenAsync(); - cmd.Connection = conn; - using (var reader = await cmd.ExecuteReaderAsync()) + if (reader.HasRows) { - if (reader.HasRows) + while (reader.Read()) { - while (reader.Read()) - { - string dbName = reader?.GetString(0); - if (!exclude.Contains(dbName)) - availableDbs.Add(dbName); - } + string dbName = reader?.GetString(0); + if (!exclude.Contains(dbName)) + availableDbs.Add(dbName); } - await reader.CloseAsync(); } - await conn.CloseAsync(); + await reader.CloseAsync(); } + await conn.CloseAsync(); } return availableDbs; } @@ -95,12 +89,10 @@ public async Task> GetAvailableDatabaseCollectionAsync(ResourceGrou try { string connectionString = resourceGroup.GetDbConnectionString(); - using (SqlConnection conn = new SqlConnection(connectionString)) - { - await conn.OpenAsync(); - await conn.CloseAsync(); - return (true, string.Empty); - } + using SqlConnection conn = new SqlConnection(connectionString); + await conn.OpenAsync(); + await conn.CloseAsync(); + return (true, string.Empty); } catch (Exception ex) { diff --git a/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs b/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs index f11fd0d..82f9cf3 100644 --- a/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs +++ b/SemanticBackup.Infrastructure/Implementations/BackupRecordRepositoryLiteDb.cs @@ -21,13 +21,13 @@ public class BackupRecordRepositoryLiteDb : IBackupRecordRepository public BackupRecordRepositoryLiteDb(IEnumerable backupRecordStatusChangedNotifiers, IContentDeliveryRecordRepository contentDeliveryRecordPersistanceService, IDatabaseInfoRepository databaseInfoRepository) { - this._db = new LiteDatabaseAsync(new ConnectionString(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "backups.db")) { Connection = ConnectionType.Shared }); + _db = new LiteDatabaseAsync(new ConnectionString(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "backups.db")) { Connection = ConnectionType.Shared }); //Init - this._db.PragmaAsync("UTC_DATE", true).GetAwaiter().GetResult(); + _db.PragmaAsync("UTC_DATE", true).GetAwaiter().GetResult(); //Proceed - this._backupRecordStatusChangedNotifiers = backupRecordStatusChangedNotifiers; - this._contentDeliveryRecordPersistanceService = contentDeliveryRecordPersistanceService; - this._databaseInfoRepository = databaseInfoRepository; + _backupRecordStatusChangedNotifiers = backupRecordStatusChangedNotifiers; + _contentDeliveryRecordPersistanceService = contentDeliveryRecordPersistanceService; + _databaseInfoRepository = databaseInfoRepository; } @@ -102,8 +102,8 @@ public async Task AddOrUpdateAsync(BackupRecord record) public async Task UpdateStatusFeedAsync(long id, string status, string message = null, long executionInMilliseconds = 0, string updateFilePath = null) { - var collection = _db.GetCollection(); - var objFound = await collection.Query().Where(x => x.Id == id).FirstOrDefaultAsync(); + ILiteCollectionAsync collection = _db.GetCollection(); + BackupRecord objFound = await collection.Query().Where(x => x.Id == id).FirstOrDefaultAsync(); if (objFound != null) { objFound.BackupStatus = status; @@ -127,8 +127,8 @@ public async Task UpdateStatusFeedAsync(long id, string status, string mes public async Task UpdateRestoreStatusFeedAsync(long id, string status, string message = null, string confirmationToken = null) { - var collection = _db.GetCollection(); - var objFound = await collection.Query().Where(x => x.Id == id).FirstOrDefaultAsync(); + ILiteCollectionAsync collection = _db.GetCollection(); + BackupRecord objFound = await collection.Query().Where(x => x.Id == id).FirstOrDefaultAsync(); if (objFound != null) { objFound.RestoreStatus = status; diff --git a/SemanticBackup.Infrastructure/Implementations/ContentDeliveryRecordRepositoryLiteDb.cs b/SemanticBackup.Infrastructure/Implementations/ContentDeliveryRecordRepositoryLiteDb.cs index f65ed8a..70f2015 100644 --- a/SemanticBackup.Infrastructure/Implementations/ContentDeliveryRecordRepositoryLiteDb.cs +++ b/SemanticBackup.Infrastructure/Implementations/ContentDeliveryRecordRepositoryLiteDb.cs @@ -17,11 +17,11 @@ public class ContentDeliveryRecordRepositoryLiteDb : IContentDeliveryRecordRepos public ContentDeliveryRecordRepositoryLiteDb(IEnumerable backupRecordStatusChangedNotifiers) { - this._db = new LiteDatabaseAsync(new ConnectionString(System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "deliveries.db")) { Connection = ConnectionType.Shared }); + _db = new LiteDatabaseAsync(new ConnectionString(System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "deliveries.db")) { Connection = ConnectionType.Shared }); //Init - this._db.PragmaAsync("UTC_DATE", true).GetAwaiter().GetResult(); + _db.PragmaAsync("UTC_DATE", true).GetAwaiter().GetResult(); //Proceed - this._backupRecordStatusChangedNotifiers = backupRecordStatusChangedNotifiers; + _backupRecordStatusChangedNotifiers = backupRecordStatusChangedNotifiers; } public async Task> GetAllByStatusAsync(string status) diff --git a/SemanticBackup.Infrastructure/Implementations/DatabaseInfoRepositoryLiteDb.cs b/SemanticBackup.Infrastructure/Implementations/DatabaseInfoRepositoryLiteDb.cs index a917a0a..3a99c7b 100644 --- a/SemanticBackup.Infrastructure/Implementations/DatabaseInfoRepositoryLiteDb.cs +++ b/SemanticBackup.Infrastructure/Implementations/DatabaseInfoRepositoryLiteDb.cs @@ -14,9 +14,9 @@ public class DatabaseInfoRepositoryLiteDb : IDatabaseInfoRepository public DatabaseInfoRepositoryLiteDb() { - this._db = new LiteDatabaseAsync(new ConnectionString(System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "databases.db")) { Connection = ConnectionType.Shared }); + _db = new LiteDatabaseAsync(new ConnectionString(System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "databases.db")) { Connection = ConnectionType.Shared }); //Init - this._db.PragmaAsync("UTC_DATE", true).GetAwaiter().GetResult(); + _db.PragmaAsync("UTC_DATE", true).GetAwaiter().GetResult(); } public async Task> GetAllAsync(string resourceGroupId) diff --git a/SemanticBackup.Infrastructure/Implementations/ResourceGroupRepositoryLiteDb.cs b/SemanticBackup.Infrastructure/Implementations/ResourceGroupRepositoryLiteDb.cs index 694f435..e0b221c 100644 --- a/SemanticBackup.Infrastructure/Implementations/ResourceGroupRepositoryLiteDb.cs +++ b/SemanticBackup.Infrastructure/Implementations/ResourceGroupRepositoryLiteDb.cs @@ -18,13 +18,13 @@ public class ResourceGroupRepositoryLiteDb : IResourceGroupRepository public ResourceGroupRepositoryLiteDb(IBackupRecordRepository backupRecordPersistanceService, IBackupScheduleRepository backupSchedulePersistanceService, IDatabaseInfoRepository databaseInfoPersistanceService) { - this._db = new LiteDatabaseAsync(new ConnectionString(System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "resources.db")) { Connection = ConnectionType.Shared }); + _db = new LiteDatabaseAsync(new ConnectionString(System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "resources.db")) { Connection = ConnectionType.Shared }); //Init - this._db.PragmaAsync("UTC_DATE", true).GetAwaiter().GetResult(); + _db.PragmaAsync("UTC_DATE", true).GetAwaiter().GetResult(); //Proceed - this._backupRecordPersistanceService = backupRecordPersistanceService; - this._backupSchedulePersistanceService = backupSchedulePersistanceService; - this._databaseInfoPersistanceService = databaseInfoPersistanceService; + _backupRecordPersistanceService = backupRecordPersistanceService; + _backupSchedulePersistanceService = backupSchedulePersistanceService; + _databaseInfoPersistanceService = databaseInfoPersistanceService; } public async Task AddAsync(ResourceGroup record) From 08d5ff52ef333152f51483172ed426f4c6e7c99e Mon Sep 17 00:00:00 2001 From: "George Njeri (Swagfin)" Date: Sat, 28 Jun 2025 02:02:52 +0300 Subject: [PATCH 6/6] chore: faster laoding times v5.2.1.1 --- .../BackgroundJobs/BackupBackgroundJob.cs | 2 +- .../BackgroundJobs/BackupBackgroundZIPJob.cs | 2 +- ...BackupRecordDeliveryDispatchBackgroundJob.cs | 2 +- ...ackupRecordDeliverySchedulerBackgroundJob.cs | 2 +- .../BackupSchedulerBackgroundJob.cs | 17 +++++------------ .../BackgroundJobs/Bots/BackupZippingBot.cs | 1 - .../Bots/InDepthDeleteDropboxBot.cs | 1 - .../Bots/InDepthDeleteObjectStorageBot.cs | 1 - .../BackgroundJobs/Bots/MySQLBackupBot.cs | 1 - .../BackgroundJobs/Bots/SQLBackupBot.cs | 1 - .../Bots/UploaderAzureStorageBot.cs | 1 - .../BackgroundJobs/Bots/UploaderDropboxBot.cs | 1 - .../BackgroundJobs/Bots/UploaderEmailSMTPBot.cs | 1 - .../BackgroundJobs/Bots/UploaderFTPBot.cs | 1 - .../BackgroundJobs/Bots/UploaderLinkGenBot.cs | 1 - .../Bots/UploaderObjectStorageBot.cs | 1 - .../BackgroundJobs/BotsManagerBackgroundJob.cs | 2 +- SemanticBackup/SemanticBackup.csproj | 4 ++-- 18 files changed, 12 insertions(+), 30 deletions(-) diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs index 6585fef..94c4b10 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs @@ -140,7 +140,7 @@ private void SetupBackgroundRemovedExpiredBackupsService(CancellationToken cance { while (!cancellationToken.IsCancellationRequested) { - await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); try { //Proceed diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs index 6a75e7b..e7358c6 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs @@ -54,7 +54,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { - await Task.Delay(7000, cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); try { //Proceed diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs index 5c5ecb3..b2a0977 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliveryDispatchBackgroundJob.cs @@ -57,7 +57,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { - await Task.Delay(10000, cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); try { //Proceed diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs index 4d32e17..d95b871 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupRecordDeliverySchedulerBackgroundJob.cs @@ -54,7 +54,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { - await Task.Delay(4000, cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); try { //Proceed diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs b/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs index c9a2c46..839b6cd 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/BackupSchedulerBackgroundJob.cs @@ -65,7 +65,7 @@ private void SetupBackgroundService(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { - await Task.Delay(3000, cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); try { //Proceed @@ -110,17 +110,10 @@ private void SetupBackgroundService(CancellationToken cancellationToken) bool addedSuccess = await _backupRecordRepository.AddOrUpdateAsync(newRecord); if (!addedSuccess) - throw new Exception("Unable to Queue Database for Backup"); - else - _logger.LogInformation($"Queueing Scheduled Backup...SUCCESS"); - //Update Schedule - bool updatedSchedule = await _backupScheduleRepository.UpdateLastRunAsync(schedule.Id, currentTimeUTC); - if (!updatedSchedule) - _logger.LogWarning("Unable to Update Scheduled Next Run"); - //Buy Some Seconds to avoid Conflict Name - await Task.Delay(new Random().Next(100)); + throw new Exception($"Unable to Queue Database for Backup : {newRecord.Name}"); + //set last run + await _backupScheduleRepository.UpdateLastRunAsync(schedule.Id, currentTimeUTC); } - } } @@ -142,7 +135,7 @@ private void SetupBackgroundNonResponsiveStopService(CancellationToken cancellat { var t = new Thread(async () => { - List statusChecks = new List { BackupRecordStatus.EXECUTING.ToString(), BackupRecordStatus.COMPRESSING.ToString(), BackupRecordDeliveryStatus.EXECUTING.ToString() }; + List statusChecks = [BackupRecordStatus.EXECUTING.ToString(), BackupRecordStatus.COMPRESSING.ToString(), BackupRecordDeliveryStatus.EXECUTING.ToString()]; int executionTimeoutInMinutes = _persistanceOptions.ExecutionTimeoutInMinutes < 1 ? 1 : _persistanceOptions.ExecutionTimeoutInMinutes; while (!cancellationToken.IsCancellationRequested) { diff --git a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/BackupZippingBot.cs b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/BackupZippingBot.cs index 2d3033f..77bd2bd 100644 --- a/SemanticBackup.Infrastructure/BackgroundJobs/Bots/BackupZippingBot.cs +++ b/SemanticBackup.Infrastructure/BackgroundJobs/Bots/BackupZippingBot.cs @@ -34,7 +34,6 @@ public async Task RunAsync(Funcnet8.0 98e83838-8ab0-44d3-a023-52d80ba01705 Linux - 5.1.6 - 5.1.6 + 5.2.1.1 + 5.2.1.1