From acce044cd3dcd5d1e46f68017e1b7da543fd8d80 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 2 May 2025 10:10:04 +0100 Subject: [PATCH 1/3] add post load --- .../PostLoad/UpdateCatalogueMetadata.cs | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs diff --git a/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs b/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs new file mode 100644 index 0000000000..8eaea1d485 --- /dev/null +++ b/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs @@ -0,0 +1,101 @@ +using FAnsi.Discovery; +using MongoDB.Driver; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.DataLoad.Triggers; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.QueryBuilding; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.DataLoad.Engine.Mutilators.PostLoad +{ + class UpdateCatalogueMetadata : IMutilateDataTables + { + + [DemandsInitialization("Update the time period associated with the dataset", DefaultValue = true)] + public bool UpdateTimePeriods { get; set; } + + [DemandsInitialization("Update the description of the dataset", DefaultValue = true)] + public bool UpdateDescription { get; set; } + + [DemandsInitialization("The Column within the data to use for temporal updates")] + public string DateColumn { get; set; } + + + [DemandsInitialization("Catalogues in Data Load to not update")] + public List CataloguestoIgnore { get; set; } + + [DemandsInitialization(@"The Text added to the description to denote the update. +Some variables are available: +%d - Todays date +%c - The number of new records +%s - The earliest date in the new records +%l - The latest date in the new records +", DefaultValue = "Update %d: Added %c Records with dates between %s and %l.")] + public string DescriptionUpdateText { get; set; } + + public void Check(ICheckNotifier notifier) + { + } + + public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage) + { + } + + public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener) + { + } + + public ExitCodeType Mutilate(IDataLoadJob job) + { + if (!UpdateTimePeriods && !UpdateDescription) return ExitCodeType.OperationNotRequired; + var catalogues = job.LoadMetadata.GetAllCatalogues().Where(c => !CataloguestoIgnore.Contains(c)); + foreach(var catalogue in catalogues) + { + var column = catalogue.CatalogueItems.Where(ci => ci.ColumnInfo.GetRuntimeName() == DateColumn).First(); + var dataLoadID = job.JobID; + var qb = new QueryBuilder(null, null); + qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), column.ColumnInfo)); + qb.AddCustomLine($"{SpecialFieldNames.DataLoadRunID} = {dataLoadID}", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE); + var sql = qb.SQL; + + var dt = new DataTable(); + dt.BeginLoadData(); + var server = catalogue.GetDistinctLiveDatabaseServer(ReusableLibraryCode.DataAccess.DataAccessContext.DataLoad, false); + var con = server.GetConnection(); + con.Open(); + using (var cmd = server.GetCommand(sql, con)) + { + using var da = server.GetDataAdapter(cmd); + da.Fill(dt); + } + dt.EndLoadData(); + if (dt.Rows.Count == 0) return ExitCodeType.OperationNotRequired; + var latestDate = dt.AsEnumerable().Max(row => DateTime.Parse(row[0].ToString())); + var earliestDate = dt.AsEnumerable().Min(row => DateTime.Parse(row[0].ToString())); + var newRecordsCount = dt.Rows.Count; + if (UpdateTimePeriods) + { + catalogue.StartDate = earliestDate; + catalogue.EndDate = latestDate; + } + if (UpdateDescription) + { + var description = catalogue.Description; + description = description + "\n\r" + DescriptionUpdateText.Replace("%d", DateTime.Now.ToString("dd/MM/yyyy")).Replace("%c", $"{newRecordsCount}").Replace("%s", $"{earliestDate.Day}/{earliestDate.Month}/{earliestDate.Year}").Replace("%l", $"{latestDate.Day}/{latestDate.Month}/{latestDate.Year}"; + } + catalogue.SaveToDatabase(); + } + + return ExitCodeType.Success; + } + } +} From a4d53cf9021db00a6d3fc67d4421e271dd85c6bc Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 2 May 2025 10:25:39 +0100 Subject: [PATCH 2/3] fix build --- .../Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs b/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs index 8eaea1d485..867df8bd66 100644 --- a/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs +++ b/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs @@ -90,7 +90,7 @@ public ExitCodeType Mutilate(IDataLoadJob job) if (UpdateDescription) { var description = catalogue.Description; - description = description + "\n\r" + DescriptionUpdateText.Replace("%d", DateTime.Now.ToString("dd/MM/yyyy")).Replace("%c", $"{newRecordsCount}").Replace("%s", $"{earliestDate.Day}/{earliestDate.Month}/{earliestDate.Year}").Replace("%l", $"{latestDate.Day}/{latestDate.Month}/{latestDate.Year}"; + description = description + "\n\r" + DescriptionUpdateText.Replace("%d", DateTime.Now.ToString("dd/MM/yyyy")).Replace("%c", $"{newRecordsCount}").Replace("%s", $"{earliestDate.Day}/{earliestDate.Month}/{earliestDate.Year}").Replace("%l", $"{latestDate.Day}/{latestDate.Month}/{latestDate.Year}"); } catalogue.SaveToDatabase(); } From d621820230ff4a13642dc17636961de0a3b349b3 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 6 May 2025 09:49:15 +0100 Subject: [PATCH 3/3] tidy up --- .../Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs b/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs index 867df8bd66..59158ca322 100644 --- a/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs +++ b/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs @@ -90,9 +90,14 @@ public ExitCodeType Mutilate(IDataLoadJob job) if (UpdateDescription) { var description = catalogue.Description; - description = description + "\n\r" + DescriptionUpdateText.Replace("%d", DateTime.Now.ToString("dd/MM/yyyy")).Replace("%c", $"{newRecordsCount}").Replace("%s", $"{earliestDate.Day}/{earliestDate.Month}/{earliestDate.Year}").Replace("%l", $"{latestDate.Day}/{latestDate.Month}/{latestDate.Year}"); + description = description + "\n\r" + DescriptionUpdateText + .Replace("%d", DateTime.Now.ToString("dd/MM/yyyy")) + .Replace("%c", $"{newRecordsCount}") + .Replace("%s", $"{earliestDate.Day}/{earliestDate.Month}/{earliestDate.Year}") + .Replace("%l", $"{latestDate.Day}/{latestDate.Month}/{latestDate.Year}"); } catalogue.SaveToDatabase(); + dt.Dispose(); } return ExitCodeType.Success;