Task/rdmp 224 dqe update #2095

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

JFriel wants to merge 43 commits into develop from task/RDMP-224-DQE-update

Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs

-Original file line number
+Diff line change
@@ Expand Up @@
             uiInstance.SetDatabaseObject(this, databaseObject);
-            if (insertIndex is not null)
+            if (insertIndex is not null && _mainDockPanel.ActivePane is not null)
             {
                 _mainDockPanel.ActivePane.SetContentIndex(floatable, (int)insertIndex);
             }
@@ Expand Down @@

CHANGELOG.md

-Original file line number
+Diff line change
@@ Expand Up @@
     ## [Unreleased]
     - Build on and target .Net 9 rather than 8
+    - Add DQE Updater Mutilator for Data Loads see [DQE Post Load runner](./Documentation/DataLoadEngine/DQEPostLoadRunner.md)
     - Simplify DB Patching Interface
     - Fix issue with Simple File Extractor pipeline component checking
@@ Expand Down @@

Documentation/DataLoadEngine/DQEPostLoadRunner.md

-Original file line number
+Diff line change
@@ -0,0 +1,12 @@
+    # DQE Post Load Runner
+    The DQE post-load runner can be used to automatically perform a DQE update once a data load completes.
+    The runner attempts to reuse any existing DQE results that have been unaffected by the data load, however this process can still be slow if the catalogue data is large and/or complex.
+    ## Requirements
+    The DQE post-load runner requires an existing DQE result to exist, otherwise it will fail.
+    ## Configuration
+    The runner makes a number of queries to the database, the timeout for these commands is configurable via the timeout option.

Rdmp.Core.Tests/DataQualityEngine/DQEPartialUpdateTests.cs

Large diffs are not rendered by default.

Rdmp.Core/CommandLine/Options/DqeOptions.cs

-Original file line number
+Diff line change
@@ Expand Up / @@ -16,4 +16,10 @@ public class DqeOptions : RDMPCommandLineOptions @@
     {
         [Option('c', "Catalogue", HelpText = "ID of the Catalogue to run the DQE on", Required = true)]
         public string Catalogue { get; set; }
+        [Option('d', "DataLoad", HelpText = "ID of the Data Load to run the DQE on. Adds new data to existing DQE results if they exist", Required = false)]
+        public string DataLoadUpdateID { get; set; }
+        [Option('t', "Timeout", HelpText = "How long(in seconds) each internal SQL command should brun for before timing out")]
+        public int CommandTimeout { get; set; }
     }

Rdmp.Core/CommandLine/Runners/DqeRunner.cs

-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ @@
     // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
     using System;
+    using Org.BouncyCastle.Security.Certificates;
     using Rdmp.Core.CommandLine.Options;
     using Rdmp.Core.Curation.Data;
     using Rdmp.Core.DataFlowPipeline;
@@ Expand All @@
             ICheckNotifier checkNotifier, GracefulCancellationToken token)
         {
             var catalogue = GetObjectFromCommandLineString<Catalogue>(repositoryLocator, _options.Catalogue);
+            int? dataLoadID = null;
+            if (_options.DataLoadUpdateID != null)
+                dataLoadID = int.Parse(_options.DataLoadUpdateID);
             var report = new CatalogueConstraintReport(catalogue, SpecialFieldNames.DataLoadRunID);
             switch (_options.Command)
             {
                 case CommandLineActivity.run:
-                    report.GenerateReport(catalogue, listener, token.AbortToken);
+                    if (dataLoadID is not null)
+                        report.UpdateReport(catalogue, (int)dataLoadID, _options.CommandTimeout, listener, token.AbortToken);
+                    else
+                        report.GenerateReport(catalogue, listener, token.AbortToken);
                     return 0;
                 case CommandLineActivity.check:
@@ Expand Down @@

Rdmp.Core/Curation/Data/Aggregation/AggregateFilter.cs

-Original file line number
+Diff line change
@@ Expand Up @@
             CopyShallowValuesTo(clone);
             return clone;
         }
-    }
+    }

Rdmp.Core/Curation/Data/ExtractionFilter.cs

-Original file line number
+Diff line change
@@ Expand Up / @@ -187,4 +187,4 @@ public override void DeleteInDatabase() @@
             base.DeleteInDatabase();
         }
-    }
+    }

Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Runtime/AttacherRuntimeTask.cs

-Original file line number
+Diff line change
@@ Expand Up / @@ -11,6 +11,7 @@ @@
     using Rdmp.Core.DataLoad.Engine.Attachers;
     using Rdmp.Core.DataLoad.Engine.Job;
     using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Arguments;
+    using Rdmp.Core.DataLoad.Modules.Attachers;
     using Rdmp.Core.Repositories;
     using Rdmp.Core.ReusableLibraryCode.Checks;
     using Rdmp.Core.ReusableLibraryCode.Progress;
@@ Expand All @@
         public AttacherRuntimeTask(IProcessTask task, RuntimeArgumentCollection args)
             : base(task, args)
         {
+            //RequestsExternalDatabaseCreation
             //All attachers must be marked as mounting stages, and therefore we can pull out the RAW Server and Name
             var mountingStageArgs = args.StageSpecificArguments;
             if (mountingStageArgs.LoadStage != LoadStage.Mounting)
@@ Expand Down @@

Rdmp.Core/DataLoad/Modules/Mutilators/DQEPostLoadRunner.cs

-Original file line number
+Diff line change
@@ Expand Up / @@ -27,6 +27,8 @@ namespace Rdmp.Core.DataLoad.Modules.Mutilators; @@
     public class DQEPostLoadRunner : IMutilateDataTables
     {
+        [DemandsInitialization("Timeout length for each query required to run the DQE update",defaultValue:50000)]
+        public int Timeout { get; set; }
         public void Check(ICheckNotifier notifier)
         {
         }
@@ Expand Down Expand Up / @@ -73,7 +75,9 @@ public ExitCodeType Mutilate(IDataLoadJob job) @@
                 DqeOptions options = new()
                 {
                     Catalogue = catalogue.ID.ToString(),
-                    Command = CommandLineActivity.run
+                    DataLoadUpdateID = job.DataLoadInfo.ID.ToString(),
+                    Command = CommandLineActivity.run,
+                    CommandTimeout = Timeout
                 };
                 var runner = RunnerFactory.CreateRunner(new ThrowImmediatelyActivator(job.RepositoryLocator), options);
                 runner.Run(job.RepositoryLocator, ThrowImmediatelyDataLoadEventListener.Quiet, new AcceptAllCheckNotifier(),
@@ Expand Down @@

Rdmp.Core/DataLoad/Triggers/DiffDatabaseDataFetcher.cs

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -10,6 +10,7 @@
  
    using System.Text;

    using FAnsi;

    using FAnsi.Discovery;

    using MongoDB.Driver;

    using Rdmp.Core.Curation.Data;

    using Rdmp.Core.Curation.Data.Spontaneous;

    using Rdmp.Core.QueryBuilding;

    @@ -113,7 +114,7 @@ public void FetchData(ICheckNotifier checkNotifier)
  
                    CheckResult.Success));

                GetInsertData(server, database, checkNotifier);

                GetUpdatetData(server, database, checkNotifier);

                GetUpdatedData(server, database, checkNotifier);

            }

            catch (Exception e)

            {

    @@ -163,7 +164,7 @@ private void GetInsertData(DiscoveredServer server, DiscoveredDatabase database,
  
        }

        private void GetUpdatetData(DiscoveredServer server, DiscoveredDatabase database, ICheckNotifier checkNotifier)

        private void GetUpdatedData(DiscoveredServer server, DiscoveredDatabase database, ICheckNotifier checkNotifier)

        {

            const string archive = "archive";

            const string zzArchive = "zzarchivezz";

    @@ -191,7 +192,8 @@ private void GetUpdatetData(DiscoveredServer server, DiscoveredDatabase database
  
    --Records which appear in the archive

    SELECT top {{0}}

    {{6}},

    {{7}}

    {{7}},

    {{8}}

    FROM    {{1}}

    CROSS APPLY

            (

    @@ -200,7 +202,7 @@ SELECT  TOP 1 {{2}}.*
  
            WHERE

             {{3}}

             order by {syntaxHelper.EnsureWrapped(SpecialFieldNames.ValidFrom)} desc

            ) {{8}}

            ) {{9}}

    where

    {{1}}.{{4}} = {{5}}";

                    break;

    @@ -214,13 +216,14 @@ SELECT  TOP 1 {{2}}.*
  
    /*Records which appear in the archive*/

    SELECT

    {{6}},

    {{7}}

    {{7}},

    {{8}}

    FROM

    {{1}}

    Join

    {{2}} {{8}} on {whereStatement.Replace(archiveTableName, archive)}

    {{2}} {{9}} on {whereStatement.Replace(archiveTableName, archive)}

     AND

     {{8}}.{{9}} = (select max({syntaxHelper.EnsureWrapped(SpecialFieldNames.ValidFrom)}) from {{2}} s where {whereStatement.Replace(archiveTableName, archive).Replace(tableName, "s")})

     {{9}}.{{10}} = (select max({syntaxHelper.EnsureWrapped(SpecialFieldNames.ValidFrom)}) from {{2}} s where {whereStatement.Replace(archiveTableName, archive).Replace(tableName, "s")})

     where

      {{1}}.{{4}} = {{5}}

    @@ -241,8 +244,9 @@ SELECT  TOP 1 {{2}}.*
  
                _dataLoadRunID, //{5}

                GetSharedColumnsSQL(tableName), //{6}

                GetSharedColumnsSQLWithColumnAliasPrefix(archive, zzArchive), //{7}

                archive, //{8}

                syntaxHelper.EnsureWrapped(SpecialFieldNames.ValidFrom)

                GetHICSpecialColumns(archive, zzArchive),//{8}

                archive, //{9}

                syntaxHelper.EnsureWrapped(SpecialFieldNames.ValidFrom) //{10}

            );

            var dtComboTable = new DataTable();

    @@ -253,11 +257,15 @@ SELECT  TOP 1 {{2}}.*
  
            //add the columns from the combo table to both views

            foreach (DataColumn col in dtComboTable.Columns)

            {

                if (!col.ColumnName.StartsWith(zzArchive, StringComparison.InvariantCultureIgnoreCase))

                {

                    Updates_New.Columns.Add(col.ColumnName, col.DataType);

                    Updates_Replaced.Columns.Add(col.ColumnName, col.DataType);

                }

            }

            Updates_Replaced.Columns.Add(SpecialFieldNames.DataLoadRunID, typeof(int));

            Updates_Replaced.Columns.Add(SpecialFieldNames.ValidFrom, typeof(DateTime));

            foreach (DataRow fromRow in dtComboTable.Rows)

            {

    @@ -272,6 +280,13 @@ SELECT  TOP 1 {{2}}.*
  
            }

        }

        private string GetHICSpecialColumns(string tableName, string columnAliasPrefix = "")

        {

            return $@"{tableName}.{SpecialFieldNames.DataLoadRunID} as {columnAliasPrefix}{SpecialFieldNames.DataLoadRunID},

    {tableName}.{SpecialFieldNames.ValidFrom} as {columnAliasPrefix}{SpecialFieldNames.ValidFrom}

    ";

        }

        private string GetSharedColumnsSQLWithColumnAliasPrefix(string tableName, string columnAliasPrefix)

        {

            var sb = new StringBuilder();

Rdmp.Core/DataQualityEngine/Data/ColumnState.cs

-Original file line number
+Diff line change
@@ Expand Up @@
                 DatabaseCommandHelper.AddParameterWithValueToCommand("@PivotCategory", cmd, pivotCategory);
                 cmd.ExecuteNonQuery();
             }
             IsCommitted = true;
         }
     }

Rdmp.Core/DataQualityEngine/Data/RowState.cs

-Original file line number
+Diff line change
@@ Expand Up / @@ -25,6 +25,17 @@ public class RowState @@
         public string PivotCategory { get; private set; }
+        public RowState(int dataLoadRunID, int correct, int missing, int wrong, int invalid,
+            string validatorXml, string pivotCategory)
+        {
+            Correct = correct;
+            Missing = missing;
+            Wrong = wrong;
+            Invalid = invalid;
+            ValidatorXML = validatorXml;
+            DataLoadRunID = dataLoadRunID;
+        }
         public RowState(DbDataReader r)
         {
             Correct = Convert.ToInt32(r["Correct"]);
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Task/rdmp 224 dqe update #2095

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Task/rdmp 224 dqe update #2095

Are you sure you want to change the base?

Uh oh!

Task/rdmp 224 dqe update #2095

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!