Skip to content

Feature/optimization patches #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
swagfin merged 4 commits into from
Jun 29, 2025
Merged

Feature/optimization patches #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f9b6676
chore: optimization patches
swagfin Jun 29, 2025
a18fac5
v5.2.1.2
swagfin Jun 29, 2025
eaba4c8
chore: logging on bots
swagfin Jun 29, 2025
1dd1cb8
chore: Added with Retry Policy
swagfin Jun 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion ...cBackup.Core/Extensions/TimeExtensions.cs → ...kup.Core/Extensions/DateTimeExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;

namespace SemanticBackup.Core
{
public static class TimeExtensions
public static class DateTimeExtensions
{
public static DateTime ConvertFromUTC(this DateTime dateTime, string timezone)
{
Expand All @@ -22,22 +23,27 @@ public static DateTime ConvertToUTC(this DateTime dateTime, string timezone)
dateTime = SetKind(dateTime);
return TimeZoneInfo.ConvertTimeToUtc(dateTime, tz);
}

private static DateTime SetKind(DateTime date) => DateTime.SpecifyKind(date, DateTimeKind.Unspecified);

public static DateTime IgnoreSeconds(this DateTime time, bool end)
{
time = new DateTime(time.Year, time.Month, time.Day, time.Hour, time.Minute, 0, 0, time.Kind);
return end ? time.AddMinutes(1) : time;
}

public static DateTime StartOfWeek(this DateTime dt, DayOfWeek startOfWeek = DayOfWeek.Monday)
{
int diff = (7 + (dt.DayOfWeek - startOfWeek)) % 7;
return dt.AddDays(-1 * diff).Date;
}

public static DateTime EndOfWeek(this DateTime dt, DayOfWeek startOfWeek = DayOfWeek.Sunday)
{
int diff = (7 + (startOfWeek - dt.DayOfWeek)) % 7;
return dt.AddDays(diff).Date;
}

public static DateTime AdjustWithTimezoneOffset(this DateTime dateTime, string timezoneOffset = "00:00")
{
try
Expand Down Expand Up @@ -80,5 +86,23 @@ public static string ToTimezoneWithOffsetString(this TimeZoneInfo timeZoneInfo)
(string timezone, string offset) timezoneWithOffset = ToTimezoneWithOffset(timeZoneInfo);
return string.Format("{0} ({1})", timezoneWithOffset.timezone, timezoneWithOffset.offset);
}

public static string ToUtcDifferenceString(this DateTime targetUtc, DateTime? compareUtc = null)
{
DateTime baseTime = compareUtc ?? DateTime.UtcNow;
TimeSpan diff = targetUtc - baseTime;
if (diff.TotalSeconds <= 0) return "Expired";

int days = diff.Days;
int hours = diff.Hours;
int minutes = diff.Minutes;

List<string> parts = [];
if (days > 0) parts.Add($"{days} day(s)");
if (hours > 0) parts.Add($"{hours}hr{(hours == 1 ? "" : "s")}");
if (minutes > 0) parts.Add($"{minutes}min");

return parts.Count > 0 ? string.Join(" ", parts) : "Expired";
}
}
}
66 changes: 27 additions & 39 deletions SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,55 +74,43 @@ private void SetupBackgroundService(CancellationToken cancellationToken)
try
{
//Proceed
List<BackupRecord> queuedBackups = await _backupRecordRepository.GetAllByStatusAsync(BackupRecordStatus.QUEUED.ToString());
if (queuedBackups != null && queuedBackups.Count > 0)
List<BackupRecord> queuedBackups = (await _backupRecordRepository.GetAllByStatusAsync(BackupRecordStatus.QUEUED.ToString())) ?? [];
List<long> scheduleToDelete = [];
foreach (BackupRecord backupRecord in queuedBackups.OrderBy(x => x.Id).ToList())
{
List<long> scheduleToDelete = [];
foreach (BackupRecord backupRecord in queuedBackups.OrderBy(x => x.RegisteredDateUTC).ToList())
_logger.LogInformation("Processing Queued Backup Record Key: #{Id}...", backupRecord.Id);
BackupDatabaseInfo backupDatabaseInfo = await _databaseInfoRepository.GetByIdAsync(backupRecord.BackupDatabaseInfoId);
if (backupDatabaseInfo == null)
{
_logger.LogInformation($"Processing Queued Backup Record Key: #{backupRecord.Id}...");
BackupDatabaseInfo backupDatabaseInfo = await _databaseInfoRepository.GetByIdAsync(backupRecord.BackupDatabaseInfoId);
if (backupDatabaseInfo == null)
_logger.LogWarning("No Database Info matches with Id: {BackupDatabaseInfoId}, Backup Database Record will be Deleted: {Id}", backupRecord.BackupDatabaseInfoId, backupRecord.Id);
scheduleToDelete.Add(backupRecord.Id);
}
else
{
//Check if valid Resource Group
ResourceGroup resourceGroup = await _resourceGroupRepository.GetByIdOrKeyAsync(backupDatabaseInfo.ResourceGroupId);
if (resourceGroup == null)
{
_logger.LogWarning($"No Database Info matches with Id: {backupRecord.BackupDatabaseInfoId}, Backup Database Record will be Deleted: {backupRecord.Id}");
_logger.LogWarning("The Database Id: {BackupDatabaseInfoId}, doesn't seem to have been assigned to a valid Resource Group Id: {ResourceGroupId}, Record will be Deleted", backupRecord.BackupDatabaseInfoId, backupDatabaseInfo.ResourceGroupId);
scheduleToDelete.Add(backupRecord.Id);
}
else
{
//Check if valid Resource Group
ResourceGroup resourceGroup = await _resourceGroupRepository.GetByIdOrKeyAsync(backupDatabaseInfo.ResourceGroupId);
if (resourceGroup == null)
{
_logger.LogWarning($"The Database Id: {backupRecord.BackupDatabaseInfoId}, doesn't seem to have been assigned to a valid Resource Group Id: {backupDatabaseInfo.ResourceGroupId}, Record will be Deleted");
scheduleToDelete.Add(backupRecord.Id);
}
if (resourceGroup.DbType.Contains("SQLSERVER"))
_botsManagerBackgroundJob.AddBot(new SQLBackupBot(backupDatabaseInfo.DatabaseName, resourceGroup, backupRecord, _providerForSQLServer));
else if (resourceGroup.DbType.Contains("MYSQL") || resourceGroup.DbType.Contains("MARIADB"))
_botsManagerBackgroundJob.AddBot(new MySQLBackupBot(backupDatabaseInfo.DatabaseName, resourceGroup, backupRecord, _providerForMySqlServer));
else
{
if (_botsManagerBackgroundJob.HasAvailableResourceGroupBotsCount(resourceGroup.Id, resourceGroup.MaximumRunningBots))
{
if (resourceGroup.DbType.Contains("SQLSERVER"))
_botsManagerBackgroundJob.AddBot(new SQLBackupBot(backupDatabaseInfo.DatabaseName, resourceGroup, backupRecord, _providerForSQLServer));
else if (resourceGroup.DbType.Contains("MYSQL") || resourceGroup.DbType.Contains("MARIADB"))
_botsManagerBackgroundJob.AddBot(new MySQLBackupBot(backupDatabaseInfo.DatabaseName, resourceGroup, backupRecord, _providerForMySqlServer));
else
throw new Exception($"No Bot is registered to Handle Database Backups of Type: {resourceGroup.DbType}");
//Finally Update Status
bool updated = await _backupRecordRepository.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.EXECUTING.ToString());
if (updated)
_logger.LogInformation($"Processing Queued Backup Record Key: #{backupRecord.Id}...SUCCESS");
else
_logger.LogWarning($"Queued for Backup but was unable to update backup record Key: #{backupRecord.Id} status");
}
else
_logger.LogInformation($"Resource Group With Id: {resourceGroup.Id} Bots are Busy, Running Bots Count: {resourceGroup.MaximumRunningBots}, waiting for available Bots....");
}

throw new Exception($"No Bot is registered to Handle Database Backups of Type: {resourceGroup.DbType}");
//Finally Update Status
_ = await _backupRecordRepository.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.EXECUTING.ToString());
}

}
//Check if Any Delete
foreach (var rm in scheduleToDelete)
await _backupRecordRepository.RemoveWithFileAsync(rm);
}
//Check if Any Delete
foreach (long backupRecordId in scheduleToDelete)
await _backupRecordRepository.RemoveWithFileAsync(backupRecordId);

}
catch (Exception ex)
Expand All @@ -146,7 +134,7 @@ private void SetupBackgroundRemovedExpiredBackupsService(CancellationToken cance
//Proceed
List<BackupRecord> expiredBackups = (await _backupRecordRepository.GetAllExpiredAsync()) ?? [];
//proceed
foreach (BackupRecord rm in expiredBackups.Take(50).ToList())
foreach (BackupRecord rm in expiredBackups.OrderBy(x => x.Id).Take(50).ToList())
{
//get relation
List<BackupRecordDelivery> rmBackupRecords = (await _deliveryRecordRepository.GetAllByBackupRecordIdAsync(rm.Id)) ?? [];
Expand Down
55 changes: 21 additions & 34 deletions SemanticBackup.Infrastructure/BackgroundJobs/BackupBackgroundZIPJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,46 +58,33 @@ private void SetupBackgroundService(CancellationToken cancellationToken)
try
{
//Proceed
List<BackupRecord> queuedBackups = await _backupRecordRepository.GetAllByStatusAsync(BackupRecordStatus.COMPLETED.ToString());
if (queuedBackups != null && queuedBackups.Count > 0)
List<BackupRecord> queuedBackups = (await _backupRecordRepository.GetAllByStatusAsync(BackupRecordStatus.COMPLETED.ToString())) ?? [];
foreach (BackupRecord backupRecord in queuedBackups.OrderBy(x => x.Id).ToList())
{
foreach (BackupRecord backupRecord in queuedBackups.OrderBy(x => x.RegisteredDateUTC).ToList())
//get valid database
BackupDatabaseInfo backupRecordDbInfo = await _databaseInfoRepository.GetByIdAsync(backupRecord.BackupDatabaseInfoId);
//Check if valid Resource Group
ResourceGroup resourceGroup = await _resourceGroupRepository.GetByIdOrKeyAsync(backupRecordDbInfo?.ResourceGroupId ?? string.Empty);
if (resourceGroup != null)
{
//get valid database
BackupDatabaseInfo backupRecordDbInfo = await _databaseInfoRepository.GetByIdAsync(backupRecord.BackupDatabaseInfoId);
//Check if valid Resource Group
ResourceGroup resourceGroup = await _resourceGroupRepository.GetByIdOrKeyAsync(backupRecordDbInfo?.ResourceGroupId ?? string.Empty);
if (resourceGroup != null)
//Use Resource Group Threads
if (resourceGroup.CompressBackupFiles)
{
//Use Resource Group Threads
if (resourceGroup.CompressBackupFiles)
{
//Check Resource Group Maximum Threads
if (_botsManagerBackgroundJob.HasAvailableResourceGroupBotsCount(resourceGroup.Id, resourceGroup.MaximumRunningBots))
{
_logger.LogInformation($"Queueing Zip Database Record Key: #{backupRecord.Id}...");
//Add to Queue
_botsManagerBackgroundJob.AddBot(new BackupZippingBot(resourceGroup.Id, backupRecord));
bool updated = await _backupRecordRepository.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.COMPRESSING.ToString());
if (updated)
_logger.LogInformation($"Queueing Zip Database Record Key: #{backupRecord.Id}...SUCCESS");
else
_logger.LogWarning($"Queued for Zipping But Failed to Update Status for Backup Record Key: #{backupRecord.Id}");
}
}
else
{
_logger.LogInformation($">> Skipping Compression for Database Record Key: #{backupRecord.Id}...");
bool updated = await _backupRecordRepository.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.READY.ToString());
if (updated)
_logger.LogInformation($">> Skipped Compression and Completed Backup Updated Record Key: #{backupRecord.Id}...SUCCESS");
else
_logger.LogWarning($"Failed to Update Status as READY for Backup Record Key: #{backupRecord.Id}");
}
_logger.LogInformation("Queueing Zip Database Record Key: #{Id}...", backupRecord.Id);
//Add to Queue
_botsManagerBackgroundJob.AddBot(new BackupZippingBot(resourceGroup.Id, backupRecord));
//Finally Update Status
_ = await _backupRecordRepository.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.COMPRESSING.ToString());
}
else
_logger.LogWarning($"The Backup Record Id: {backupRecord.Id}, doesn't seem to have been assigned to a valid Resource Group, Zipping Skipped");
{
_logger.LogInformation(">> Skipping Compression for Database Record Key: #{Id}...", backupRecord.Id);
//Finally Update Status
_ = await _backupRecordRepository.UpdateStatusFeedAsync(backupRecord.Id, BackupRecordStatus.READY.ToString());
}
}
else
_logger.LogWarning("The Backup Record Id: {Id}, doesn't seem to have been assigned to a valid Resource Group, Zipping Skipped", backupRecord.Id);
}
}
catch (Exception ex)
Expand Down
Loading
Loading