From da6f3bf0f488f705b50fa922c627795171ede44b Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 25 Apr 2025 09:16:02 +0100 Subject: [PATCH 001/142] start to add dataste providers --- .../ExecuteCommandCreateDatasetTests.cs | 2 +- .../ExecuteCommandDeleteDatasetTest.cs | 10 +-- ...ecuteCommandLinkCatalogueToDatasetTests.cs | 8 +- ...teCommandLinkCoulmnInfoWithDatasetTests.cs | 6 +- .../ExecuteCommandCreateDataset.cs | 3 +- .../ExecuteCommandDeleteDataset.cs | 4 +- .../ExecuteCommandLinkCatalogueToDataset.cs | 4 +- .../ExecuteCommandLinkColumnInfoToDataset.cs | 4 +- .../Data/Datasets/CatalogueDatasetLinkage.cs | 59 +++++++++++++ .../Curation/Data/{ => Datasets}/Dataset.cs | 34 +++++++- .../Datasets/DatasetProviderConfiguration.cs | 83 +++++++++++++++++++ .../Curation/Data/{ => Datasets}/IDataset.cs | 19 ++++- .../Data/Datasets/IDatasetProvider.cs | 24 ++++++ .../Datasets/IDatasetProviderConfiguration.cs | 37 +++++++++ .../Data/Datasets/InternalDatasetProvider.cs | 29 +++++++ .../Curation/Data/Datasets/PluginDataset.cs | 17 ++++ .../Data/Datasets/PluginDatasetProvider.cs | 39 +++++++++ .../up/091_AddDatasetUpdates.sql | 44 ++++++++++ .../Dataset/DatasetConfigurationUICommon.cs | 2 +- .../Icons/IconProvision/CatalogueIcons.resx | 9 ++ .../IconProvision/CatalogueIcons.zh-Hans.resx | 9 ++ Rdmp.Core/Icons/IconProvision/RDMPConcept.cs | 5 +- Rdmp.Core/Providers/CatalogueChildProvider.cs | 29 +++++-- .../Providers/DataExportChildProvider.cs | 1 + Rdmp.Core/Providers/ICoreChildProvider.cs | 5 +- .../AllDatasetProviderConfigurationsNode.cs | 6 ++ Rdmp.Core/Rdmp.Core.csproj | 1 + .../WordDataReleaseFileGenerator.cs | 4 +- .../Reports/ExtractionTime/WordDataWriter.cs | 4 +- .../Collections/ConfigurationsCollectionUI.cs | 3 +- .../ExecuteCommandDeleteDatasetUI.cs | 2 +- .../ExecuteCommandLinkCatalogueToDatasetUI.cs | 1 + ...ExecuteCommandLinkColumnInfoToDataSetUI.cs | 1 + .../ProposeExecutionWhenTargetIsDataset.cs | 2 +- Rdmp.UI/MainFormUITabs/CatalogueItemUI.cs | 1 + Rdmp.UI/MainFormUITabs/CatalogueUI.cs | 1 + Rdmp.UI/Menus/DatasetMenu.cs | 2 +- .../SubComponents/DatasetConfigurationUI.cs | 1 + 38 files changed, 471 insertions(+), 44 deletions(-) create mode 100644 Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs rename Rdmp.Core/Curation/Data/{ => Datasets}/Dataset.cs (71%) create mode 100644 Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs rename Rdmp.Core/Curation/Data/{ => Datasets}/IDataset.cs (76%) create mode 100644 Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql create mode 100644 Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs index 3a7169231c..f0c786c31a 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs @@ -26,7 +26,7 @@ public void TestDatasetCreationOKExtendedParameters() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset2","somedoi","some source"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset2" && ds.DigitalObjectIdentifier == "somedoi" && ds.Source == "some source"); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset2" && ds.DigitalObjectIdentifier == "somedoi" && ds.Source == "some source"); Assert.That(founddataset,Is.Not.Null); } } diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs index 8982ec5ff7..155e2442e0 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs @@ -13,20 +13,20 @@ public void TestDeleteExistingDataset() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Has.Length.EqualTo(1)); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset"); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Has.Length.EqualTo(1)); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset"); var delCmd = new ExecuteCommandDeleteDataset(GetMockActivator(), founddataset); Assert.DoesNotThrow(() => delCmd.Execute()); - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); } [Test] public void TestDeleteNonExistantDataset() { - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); var founddataset = new Core.Curation.Data.Dataset(); var delCmd = new ExecuteCommandDeleteDataset(GetMockActivator(), founddataset); Assert.Throws(() => delCmd.Execute()); - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); } } \ No newline at end of file diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs index 89b7dcb6b1..330e1e3d46 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs @@ -34,7 +34,7 @@ public void TestLinkCatalogueToDataset() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static c => c.Name == "Dataset1"); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), foundCatalogue, founddataset); Assert.DoesNotThrow(linkCmd.Execute); @@ -72,7 +72,7 @@ public void TestLinkCatalogueToDatasetNotAll() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(c => c.Name == "Dataset1"); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), foundCatalogue, founddataset, false); Assert.DoesNotThrow(linkCmd.Execute); @@ -94,7 +94,7 @@ public void TestLinkCatalogueToDatasetBadCatalogue() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), null, founddataset, false); Assert.Throws(linkCmd.Execute); } @@ -104,7 +104,7 @@ public void TestLinkCatalogueToDatasetBadDataset() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository,"catalogue"), null, false); Assert.Throws(linkCmd.Execute); } diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs index 02848abedf..30dcb53432 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs @@ -33,7 +33,7 @@ public void TestLinkColumnInfoToDataset() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), _c1, founddataset); Assert.DoesNotThrow(() => linkCmd.Execute()); var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(); @@ -67,7 +67,7 @@ public void TestLinkColumnInfoToDatasetNotAll() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), _c1, founddataset, false); Assert.DoesNotThrow(() => linkCmd.Execute()); var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().Where(ci => _cata1.CatalogueItems.Contains(ci)); @@ -87,7 +87,7 @@ public void TestLinkCatalogueToDatasetBadColumnInfo() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), null, founddataset, false); Assert.Throws(() => linkCmd.Execute()); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs index c04a7929c8..acee8b5f45 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs @@ -1,5 +1,4 @@ using Rdmp.Core.Curation.Data; - namespace Rdmp.Core.CommandExecution.AtomicCommands; public class ExecuteCommandCreateDataset : BasicCommandExecution @@ -24,7 +23,7 @@ public ExecuteCommandCreateDataset(IBasicActivateItems activator, [DemandsInitia public override void Execute() { base.Execute(); - var dataset = new Curation.Data.Dataset(BasicActivator.RepositoryLocator.CatalogueRepository, _name) { DigitalObjectIdentifier = _doi, Source = _source }; + var dataset = new Rdmp.Core.Curation.Data.Datasets.Dataset(BasicActivator.RepositoryLocator.CatalogueRepository, _name) { DigitalObjectIdentifier = _doi, Source = _source }; dataset.SaveToDatabase(); _activator.Publish(dataset); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs index 734a81c431..8f65b79385 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs @@ -4,9 +4,9 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public sealed class ExecuteCommandDeleteDataset: BasicCommandExecution { - private readonly Curation.Data.Dataset _dataset; + private readonly Curation.Data.Datasets.Dataset _dataset; private readonly IBasicActivateItems _activator; -public ExecuteCommandDeleteDataset(IBasicActivateItems activator, [DemandsInitialization("The Dataset to delete")]Curation.Data.Dataset dataset) +public ExecuteCommandDeleteDataset(IBasicActivateItems activator, [DemandsInitialization("The Dataset to delete")]Curation.Data.Datasets.Dataset dataset) { _dataset = dataset; _activator = activator; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs index 683d910a14..4e74fae767 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs @@ -13,9 +13,9 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public sealed class ExecuteCommandLinkCatalogueToDataset : BasicCommandExecution { private readonly Catalogue _catalogue; - private readonly Curation.Data.Dataset _dataset; + private readonly Curation.Data.Datasets.Dataset _dataset; private readonly bool _linkAll; - public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [DemandsInitialization("The catalogue To link")]Catalogue catalogue, [DemandsInitialization("The dataset to link to")]Curation.Data.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) + public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [DemandsInitialization("The catalogue To link")]Catalogue catalogue, [DemandsInitialization("The dataset to link to")]Curation.Data.Datasets.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) { _catalogue = catalogue; _dataset = dataset; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs index 7eac784215..57034e3851 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs @@ -13,9 +13,9 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public sealed class ExecuteCommandLinkColumnInfoToDataset : BasicCommandExecution { private readonly ColumnInfo _columnInfo; - private readonly Curation.Data.Dataset _dataset; + private readonly Curation.Data.Datasets.Dataset _dataset; private readonly bool _linkAll; - public ExecuteCommandLinkColumnInfoToDataset(IBasicActivateItems activator, [DemandsInitialization("The column to link")] ColumnInfo columnInfo, [DemandsInitialization("The dataset to link to")] Curation.Data.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) + public ExecuteCommandLinkColumnInfoToDataset(IBasicActivateItems activator, [DemandsInitialization("The column to link")] ColumnInfo columnInfo, [DemandsInitialization("The dataset to link to")] Curation.Data.Datasets.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) { _columnInfo = columnInfo; _dataset = dataset; diff --git a/Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs b/Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs new file mode 100644 index 0000000000..f255cc6c2f --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs @@ -0,0 +1,59 @@ +// Copyright (c) The University of Dundee 2025-2025 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.Repositories; +using System.Collections.Generic; +using System.Data.Common; + +namespace Rdmp.Core.Curation.Data.Datasets +{ + public class CatalogueDatasetLinkage: DatabaseEntity + { + private int _catalogueID; + private int _datasetID; + private ICatalogueRepository _repository; + + #region Relationships + [NoMappingToDatabase] + public Catalogue Catalogue => _repository.GetObjectByID(_catalogueID); + + + [NoMappingToDatabase] + public Dataset Dataset => _repository.GetObjectByID(_datasetID); + + #endregion + + public int Catalogue_ID { get => _catalogueID; set => SetField(ref _catalogueID, value); } + + public int Dataset_ID { get => _datasetID; set => SetField(ref _datasetID, value); } + + public bool Autoupdate { get; set; } + + public CatalogueDatasetLinkage() { } + + public CatalogueDatasetLinkage(ICatalogueRepository repository, Catalogue catalogue, Dataset dataset, bool autoupdate = false) + { + _repository = repository; + _catalogueID = catalogue.ID; + _datasetID = dataset.ID; + repository.InsertAndHydrate(this, new Dictionary + { + {"Catalogue_ID", catalogue.ID }, + { "Dataset_ID", dataset.ID}, + {"Autoupdate", autoupdate==true?1:0 } + }); + } + + public CatalogueDatasetLinkage(ICatalogueRepository repository, DbDataReader r) : base(repository, r) + { + _repository = repository; + _catalogueID = int.Parse(r["Catalogue_ID"].ToString()); + _datasetID = int.Parse(r["Dataset_ID"].ToString()); + Autoupdate = r["Autoupdate"].ToString() != "0"; + } + } +} diff --git a/Rdmp.Core/Curation/Data/Dataset.cs b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs similarity index 71% rename from Rdmp.Core/Curation/Data/Dataset.cs rename to Rdmp.Core/Curation/Data/Datasets/Dataset.cs index 5623568db7..706de02b38 100644 --- a/Rdmp.Core/Curation/Data/Dataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs @@ -11,17 +11,21 @@ using System.Data.Common; using Rdmp.Core.MapsDirectlyToDatabaseTable; using System.Diagnostics.CodeAnalysis; +using System.Linq; -namespace Rdmp.Core.Curation.Data; +namespace Rdmp.Core.Curation.Data.Datasets; /// -public sealed class Dataset : DatabaseEntity, IDataset, IHasFolder +public class Dataset : DatabaseEntity, IDataset, IHasFolder { private string _name; private string _digitalObjectIdentifier; private string _source; private string _folder = FolderHelper.Root; + private string _type = null; + private string _url = null; + private int? _providerId = null; /// [DoNotImportDescriptions] @@ -53,8 +57,28 @@ public string Source set => SetField(ref _source, value); } + public string Type + { + get => _type; + set => SetField(ref _type, value); + } + public string Url + { + get => _url; + set => SetField(ref _url, value); + } + + public int? Provider_ID + { + get => _providerId; + set => SetField(ref _providerId, value); + } public override string ToString() => Name; + public List GetLinkedCatalogues() + { + return CatalogueRepository.GetAllObjectsWhere("Dataset_ID", this.ID).Select(l => l.Catalogue).Distinct().ToList(); + } public Dataset(ICatalogueRepository catalogueRepository, string name) { @@ -75,5 +99,11 @@ internal Dataset(ICatalogueRepository repository, DbDataReader r) DigitalObjectIdentifier = r["DigitalObjectIdentifier"].ToString(); if (r["Source"] != DBNull.Value) Source = r["Source"].ToString(); + if (r["Type"] != DBNull.Value) + Type = r["Type"].ToString(); + if (r["Url"] != DBNull.Value) + Url = r["Url"].ToString(); + if (r["Provider_ID"] != DBNull.Value) + Provider_ID = int.Parse(r["Provider_ID"].ToString()); } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs new file mode 100644 index 0000000000..125aaeb3c5 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs @@ -0,0 +1,83 @@ +using MongoDB.Driver; +using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets; + +/// +public class DatasetProviderConfiguration : DatabaseEntity, IDatasetProviderConfiguration +{ + + private string _type; + private string _name; + private string _url; + private int _dataAccessCredentials; + private string _organisationId; + + public DatasetProviderConfiguration() { } + + public DatasetProviderConfiguration(ICatalogueRepository repository, string name, string type, string url, int dataAccessCredentialsID, string organisationId) + { + repository.InsertAndHydrate(this, new Dictionary + { + {"Name",name }, + {"Type",type}, + {"Url",url}, + {"DataAccessCredentials_ID",dataAccessCredentialsID }, + {"Organisation_ID", organisationId } + }); + } + + internal DatasetProviderConfiguration(ICatalogueRepository repository, DbDataReader r) : base(repository, r) + { + Name = r["Name"].ToString(); + Type = r["Type"].ToString(); + Url = r["Url"].ToString(); + DataAccessCredentials_ID = int.Parse(r["DataAccessCredentials_ID"].ToString()); + Organisation_ID = r["Organisation_ID"].ToString(); + } + + + public override string ToString() + { + return _name; + } + + public string Name + { + get => _name; + set => SetField(ref _name, value); + } + + + public string Type + { + get => _type; + set => SetField(ref _type, value); + } + + + public string Url + { + get => _url; + set => SetField(ref _url, value); + } + + public int DataAccessCredentials_ID + { + get => _dataAccessCredentials; + set => SetField(ref _dataAccessCredentials, value); + } + + public string Organisation_ID + { + get => _organisationId; + set => SetField(ref _organisationId, value); + } +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/IDataset.cs b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs similarity index 76% rename from Rdmp.Core/Curation/Data/IDataset.cs rename to Rdmp.Core/Curation/Data/Datasets/IDataset.cs index a7853f2994..c0b91946b6 100644 --- a/Rdmp.Core/Curation/Data/IDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs @@ -7,7 +7,7 @@ using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Repositories; -namespace Rdmp.Core.Curation.Data; +namespace Rdmp.Core.Curation.Data.Datasets; /// /// The core of datasets within RDMP. @@ -20,6 +20,23 @@ public interface IDataset: IMapsDirectlyToDatabaseTable /// ICatalogueRepository CatalogueRepository { get; } + /// + /// The Name of the Dataset + /// string Name { get; } + + /// + /// The (optional) DOI of the dataset + /// string DigitalObjectIdentifier { get; } + + /// + /// The Source of the Dataset e.g. Pure, HDR + /// + string Type { get; } + + /// + /// The URL to access the dataset + /// + string Url { get; } } diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs new file mode 100644 index 0000000000..f420575814 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; + +namespace Rdmp.Core.Datasets; +/// +/// Base interface for dataset providers to impliment +/// +public interface IDatasetProvider +{ + + /// + /// Fetch known datasets + /// + /// + List FetchDatasets(); + + + /// + /// Fetch a specific dataset by its ID + /// + /// + /// + Curation.Data.Datasets.Dataset FetchDatasetByID(int id); + +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs new file mode 100644 index 0000000000..0b7fb5a9b2 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs @@ -0,0 +1,37 @@ +using Rdmp.Core.MapsDirectlyToDatabaseTable; + +namespace Rdmp.Core.Curation.Data.Datasets; + + +/// +/// Remords configuration for accessing external dataset providers, such as PURE +/// +public interface IDatasetProviderConfiguration : IMapsDirectlyToDatabaseTable +{ + /// + /// The assembly name of the provider this configuration is used for + /// + public string Type { get; } + + /// + /// A friendly name for the configuration + /// + public string Name { get; } + + /// + /// The API url + /// + public string Url { get; } + + /// + /// Reference to which credentials should be used to access the API + /// + public int DataAccessCredentials_ID { get; } + + /// + /// The organisation ID to use with the remote provider + /// + public string Organisation_ID { get; } + + +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs new file mode 100644 index 0000000000..f3641d86ff --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs @@ -0,0 +1,29 @@ +using Rdmp.Core.CommandExecution; +using System.Collections.Generic; +using System.Linq; + +namespace Rdmp.Core.Datasets; + +/// +/// Provider for internal datasets +/// +public class InternalDatasetProvider : IDatasetProvider +{ + private readonly IBasicActivateItems _activator; + public InternalDatasetProvider(IBasicActivateItems activator) + { + _activator = activator; + } + + /// + public Curation.Data.Datasets.Dataset FetchDatasetByID(int id) + { + return _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", id).FirstOrDefault(); + } + + /// + public List FetchDatasets() + { + return _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); + } +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs new file mode 100644 index 0000000000..11eb14104a --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs @@ -0,0 +1,17 @@ +using Rdmp.Core.Repositories; +using System.Data.Common; + + +namespace Rdmp.Core.Datasets +{ + /// + /// Base class to allow all plugin dataset types to be based off + /// + public class PluginDataset : Rdmp.Core.Curation.Data.Datasets.Dataset + { + public PluginDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name) { } + public PluginDataset() { } + + public PluginDataset(ICatalogueRepository repository, DbDataReader r) : base(repository, r) { } + } +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs new file mode 100644 index 0000000000..958126e86e --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -0,0 +1,39 @@ +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Datasets; + +public abstract class PluginDatasetProvider : IDatasetProvider +{ + protected DatasetProviderConfiguration Configuration { get; } + protected ICatalogueRepository Repository { get; } + protected IBasicActivateItems Activator { get; } + + protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration) + { + Configuration = configuration; + Activator = activator; + Repository = activator.RepositoryLocator.CatalogueRepository; + } + + public abstract Curation.Data.Datasets.Dataset FetchDatasetByID(int id); + + public abstract List FetchDatasets(); + + public abstract void AddExistingDataset(string name, string url); + + public abstract Curation.Data.Datasets.Dataset AddExistingDatasetWithReturn(string name, string url); + + public abstract Curation.Data.Datasets.Dataset Create(Catalogue catalogue); + + public abstract void Update(string uuid, PluginDataset datasetUpdates); + public abstract void UpdateUsingCatalogue(PluginDataset dataset, Catalogue catalogue); + +} \ No newline at end of file diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql new file mode 100644 index 0000000000..a9fb77d75e --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql @@ -0,0 +1,44 @@ +--Version: 8.4.4 +--Description: Add new metadata fields for catalogues +if not exists (select 1 from sys.tables where name = 'CatalogueDatasetLinkage') +BEGIN +CREATE TABLE [dbo].[CatalogueDatasetLinkage]( + [ID] [int] IDENTITY(1,1) NOT NULL, + [Catalogue_ID] [int] NOT NULL, + [Dataset_ID] [int] NOT NULL, + [Autoupdate] [int] NOT NULL DEFAULT 0, + CONSTRAINT FK_CatalogueDatasetLinkage_Catalogue_ID FOREIGN KEY (Catalogue_ID) REFERENCES Catalogue(ID) ON DELETE CASCADE, + CONSTRAINT FK_CatalogueDatasetLinkage_Dataset_ID FOREIGN KEY (Dataset_ID) REFERENCES Dataset(ID) ON DELETE CASCADE, +CONSTRAINT [PK_CatalogueDatasetLinkage] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO + +if not exists (select 1 from sys.tables where name = 'DatasetProviderConfiguration') +BEGIN +CREATE TABLE [dbo].[DatasetProviderConfiguration]( + [ID] [int] IDENTITY(1,1) NOT NULL, + Type [nvarchar](256) NOT NULL, + Url [nvarchar](500) NOT NULL, + Name [nvarchar](256) NOT NULL, + DataAccessCredentials_ID [int] NOT NULL, + Organisation_ID [nvarchar](256) NOT NULL + CONSTRAINT FK_DatasetProviderConfiguration_DataAccessCredentials FOREIGN KEY (DataAccessCredentials_ID) REFERENCES DataAccessCredentials(ID) ON DELETE CASCADE, +CONSTRAINT [PK_DatasetProviderConfiguration] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO +if not exists (select 1 from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME='Dataset' and COLUMN_NAME='Type') +BEGIN +ALTER TABLE [dbo].[Dataset] +ADD [Type] [varchar](256) NULL, + [Url] [varchar](256) NULL, + FOREIGN KEY (Provider_ID) REFERENCES DatasetProviderConfiguration(ID) +END +GO \ No newline at end of file diff --git a/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs b/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs index d51360e1a2..07ee42c59f 100644 --- a/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs +++ b/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs @@ -20,7 +20,7 @@ public class DatasetConfigurationUICommon /// public IBasicActivateItems Activator; - public Curation.Data.Dataset Dataset; + public Curation.Data.Datasets.Dataset Dataset; public DatasetConfigurationUICommon() diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index db264af676..09025ba357 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -784,4 +784,13 @@ ..\StandardRegex31.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllDatasetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllDatasetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx index 7d3f1ac339..a3c0205507 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx @@ -121,4 +121,13 @@ ..\zh-Hans\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + + + + + + \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs index b3988b6401..3cc2ecbf10 100644 --- a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs +++ b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs @@ -213,5 +213,8 @@ public enum RDMPConcept RegexRedaction, RegexRedactionConfiguration, RegexRedactionKey, - AllRegexRedactionConfigurationsNode + AllRegexRedactionConfigurationsNode, + DatasetProviderConfiguration, + AllDatasetProviderConfigurationsNode, + PluginDataset } \ No newline at end of file diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 16ae2f55ee..221bc26da3 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -17,6 +17,7 @@ using Rdmp.Core.Curation.Data.Cohort.Joinables; using Rdmp.Core.Curation.Data.Dashboarding; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Pipelines; @@ -66,7 +67,7 @@ public class CatalogueChildProvider : ICoreChildProvider //Catalogue side of things public Catalogue[] AllCatalogues { get; set; } - public Curation.Data.Dataset[] AllDatasets { get; set; } + public Curation.Data.Datasets.Dataset[] AllDatasets { get; set; } public Dictionary AllCataloguesDictionary { get; private set; } public SupportingDocument[] AllSupportingDocuments { get; set; } @@ -143,7 +144,7 @@ public class CatalogueChildProvider : ICoreChildProvider public AllPermissionWindowsNode AllPermissionWindowsNode { get; set; } public FolderNode LoadMetadataRootFolder { get; set; } - public FolderNode DatasetRootFolder { get; set; } + public FolderNode DatasetRootFolder { get; set; } public FolderNode CohortIdentificationConfigurationRootFolder { get; set; } public FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; set; } @@ -209,8 +210,10 @@ public class CatalogueChildProvider : ICoreChildProvider public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; set; } public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; set; } + public AllDatasetProviderConfigurationsNode AllDatasetProviderConfigurationsNode { get; set; } + public DatasetProviderConfiguration[] DatasetProviderConfigurations { get; set; } - public HashSet OrphanAggregateConfigurations; + public HashSet OrphanAggregateConfigurations; public AggregateConfiguration[] TemplateAggregateConfigurations; @@ -253,7 +256,7 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] AllCatalogues = GetAllObjects(repository); AllCataloguesDictionary = AllCatalogues.ToDictionaryEx(i => i.ID, o => o); - AllDatasets = GetAllObjects(repository); + AllDatasets = GetAllObjects(repository); AllLoadMetadatas = GetAllObjects(repository); AllLoadMetadataLinkage = GetAllObjects(repository); @@ -454,8 +457,11 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); AddChildren(AllRegexRedactionConfigurationsNode); + AllDatasetProviderConfigurationsNode = new AllDatasetProviderConfigurationsNode(); + DatasetProviderConfigurations = GetAllObjects(repository); + AddChildren(AllDatasetProviderConfigurationsNode); - AllDatasets = GetAllObjects(repository); + AllDatasets = GetAllObjects(repository); AllDatasetsNode = new AllDatasetsNode(); AddChildren(AllDatasetsNode); @@ -606,7 +612,14 @@ private void AddChildren(AllPluginsNode allPluginsNode) AddToDictionaries(children, descendancy); } - private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) + private void AddChildren(AllDatasetProviderConfigurationsNode allDatasetConfigurationNode) + { + var children = new HashSet(DatasetProviderConfigurations); + var descendancy = new DescendancyList(allDatasetConfigurationNode); + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) { var children = new HashSet(AllRegexRedactionConfigurations); var descendancy = new DescendancyList(allRegexRedactionConfigurationsNode); @@ -864,7 +877,7 @@ private void AddChildren(FolderNode folder, DescendancyList descen ); } - private void AddChildren(FolderNode folder, DescendancyList descendancy) + private void AddChildren(FolderNode folder, DescendancyList descendancy) { foreach (var child in folder.ChildFolders) //add subfolder children @@ -897,7 +910,7 @@ private void AddChildren(FolderNode folder, D ); } - private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) + private void AddChildren(Curation.Data.Datasets.Dataset lmd, DescendancyList descendancy) { var childObjects = new List(); AddToDictionaries(new HashSet(childObjects), descendancy); diff --git a/Rdmp.Core/Providers/DataExportChildProvider.cs b/Rdmp.Core/Providers/DataExportChildProvider.cs index df0266f273..286ee20968 100644 --- a/Rdmp.Core/Providers/DataExportChildProvider.cs +++ b/Rdmp.Core/Providers/DataExportChildProvider.cs @@ -820,6 +820,7 @@ public override void UpdateTo(ICoreChildProvider other) AllDatasetsNode = dxOther.AllDatasetsNode; AllRegexRedactionConfigurations = dxOther.AllRegexRedactionConfigurations; AllRegexRedactionConfigurationsNode = dxOther.AllRegexRedactionConfigurationsNode; + AllDatasetProviderConfigurationsNode = dxOther.AllDatasetProviderConfigurationsNode; } } diff --git a/Rdmp.Core/Providers/ICoreChildProvider.cs b/Rdmp.Core/Providers/ICoreChildProvider.cs index 911b02a738..1240851120 100644 --- a/Rdmp.Core/Providers/ICoreChildProvider.cs +++ b/Rdmp.Core/Providers/ICoreChildProvider.cs @@ -44,12 +44,12 @@ public interface ICoreChildProvider : IChildProvider JoinableCohortAggregateConfigurationUse[] AllJoinUses { get; set; } FolderNode CatalogueRootFolder { get; } - FolderNode DatasetRootFolder { get; } + FolderNode DatasetRootFolder { get; } FolderNode LoadMetadataRootFolder { get; } FolderNode CohortIdentificationConfigurationRootFolder { get; } FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; } Catalogue[] AllCatalogues { get; } - Curation.Data.Dataset[] AllDatasets { get; } + Curation.Data.Datasets.Dataset[] AllDatasets { get; } Dictionary AllCataloguesDictionary { get; } ExternalDatabaseServer[] AllExternalServers { get; } @@ -84,6 +84,7 @@ public interface ICoreChildProvider : IChildProvider AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; } + AllDatasetProviderConfigurationsNode AllDatasetProviderConfigurationsNode { get; } AllObjectSharingNode AllObjectSharingNode { get; } ObjectImport[] AllImports { get; } ObjectExport[] AllExports { get; } diff --git a/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs b/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs new file mode 100644 index 0000000000..e37bcdc7f0 --- /dev/null +++ b/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs @@ -0,0 +1,6 @@ +namespace Rdmp.Core.Providers.Nodes; + +public class AllDatasetProviderConfigurationsNode : SingletonNode +{ + public AllDatasetProviderConfigurationsNode() : base("Dataset Provider Configurations") { } +} \ No newline at end of file diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index e77943c6f5..b2433be2ba 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -256,6 +256,7 @@ + diff --git a/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs b/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs index 0ccb871060..de05db936a 100644 --- a/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs +++ b/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs @@ -162,7 +162,7 @@ private void CreateCohortDetailsTable(XWPFDocument document) } [NotNull] - private string getDOI([NotNull] Curation.Data.Dataset ds) + private string getDOI([NotNull] Curation.Data.Datasets.Dataset ds) { return !string.IsNullOrWhiteSpace(ds.DigitalObjectIdentifier) ? $" (DOI: {ds.DigitalObjectIdentifier})" : ""; } @@ -188,7 +188,7 @@ private void CreateFileSummary(XWPFDocument document) SetTableCell(table, tableLine, 0, extractableDataset.ToString()); var linkedDatasets = extractableDataset.Catalogue.CatalogueItems.Select(static c => c.ColumnInfo).Where(ci => ci.Dataset_ID != null).Distinct().Select(ci => ci.Dataset_ID); - var datasets = _repository.CatalogueRepository.GetAllObjects().Where(d => linkedDatasets.Contains(d.ID)).ToList(); + var datasets = _repository.CatalogueRepository.GetAllObjects().Where(d => linkedDatasets.Contains(d.ID)).ToList(); var datasetString = string.Join("",datasets.Select(ds=> $"{ds.Name} {getDOI(ds)}, {Environment.NewLine}")); SetTableCell(table, tableLine, 1, result.FiltersUsed); SetTableCell(table, tableLine, 2, filename); diff --git a/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs b/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs index 45806e2f09..cc2384d162 100644 --- a/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs +++ b/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs @@ -57,7 +57,7 @@ public WordDataWriter(ExtractionPipelineUseCase executer) [NotNull] - private static string GetDoi([NotNull] Curation.Data.Dataset ds) + private static string GetDoi([NotNull] Curation.Data.Datasets.Dataset ds) { return !string.IsNullOrWhiteSpace(ds.DigitalObjectIdentifier) ? $" (DOI: {ds.DigitalObjectIdentifier})" : ""; } @@ -163,7 +163,7 @@ public void GenerateWordFile() if (foundDatasets.Count > 0) { - var datasets = Executer.Source.Request.Catalogue.Repository.GetAllObjects().ToList(); + var datasets = Executer.Source.Request.Catalogue.Repository.GetAllObjects().ToList(); var datasetString = string.Join(", ", foundDatasets diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index 60bf6b0a6f..c75dc69052 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -4,9 +4,9 @@ using Rdmp.UI.ItemActivation; using Rdmp.UI.Refreshing; using System.Linq; -using Rdmp.Core.Curation.Data; using Rdmp.Core.Providers.Nodes; using Rdmp.Core.Curation.DataHelper.RegexRedaction; +using Rdmp.Core.Curation.Data.Datasets; namespace Rdmp.UI.Collections; @@ -43,6 +43,7 @@ public override void SetItemActivator(IActivateItems activator) CommonTreeFunctionality.WhitespaceRightClickMenuCommandsGetter = e => GetWhitespaceRightClickMenu(); Activator.RefreshBus.EstablishLifetimeSubscription(this); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetsNode); + tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetProviderConfigurationsNode); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllRegexRedactionConfigurationsNode); tlvConfigurations.Refresh(); } diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandDeleteDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandDeleteDatasetUI.cs index 90803cb3a3..e0a1117bef 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandDeleteDatasetUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandDeleteDatasetUI.cs @@ -1,4 +1,4 @@ -using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using Rdmp.UI.ItemActivation; diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDatasetUI.cs index 96657cd43c..41b09f77c3 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDatasetUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDatasetUI.cs @@ -7,6 +7,7 @@ using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataSetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataSetUI.cs index d59809fc9e..2acad8ddd8 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataSetUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataSetUI.cs @@ -7,6 +7,7 @@ using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs index 7bc681d63e..22f816d991 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs @@ -5,7 +5,7 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using Rdmp.Core.CommandExecution; -using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.UI.ItemActivation; using Rdmp.UI.SubComponents; diff --git a/Rdmp.UI/MainFormUITabs/CatalogueItemUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueItemUI.cs index 983f0b20c5..949a72e78f 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueItemUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueItemUI.cs @@ -11,6 +11,7 @@ using Rdmp.Core; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.UI.ItemActivation; using Rdmp.UI.Rules; using Rdmp.UI.ScintillaHelper; diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs index 16b3bf53d5..46146327c1 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs @@ -13,6 +13,7 @@ using System.Windows.Forms; using Rdmp.Core; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Icons.IconProvision; using Rdmp.UI.ItemActivation; using Rdmp.UI.Rules; diff --git a/Rdmp.UI/Menus/DatasetMenu.cs b/Rdmp.UI/Menus/DatasetMenu.cs index 5b833b525f..a8d734ce86 100644 --- a/Rdmp.UI/Menus/DatasetMenu.cs +++ b/Rdmp.UI/Menus/DatasetMenu.cs @@ -3,7 +3,7 @@ // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see .using Rdmp.Core.Curation.Data; -using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.UI.CommandExecution.AtomicCommands; namespace Rdmp.UI.Menus; diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs b/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs index dc7315fb89..aa44e9aab7 100644 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs @@ -7,6 +7,7 @@ using Rdmp.Core.Dataset; using Rdmp.Core.Curation.Data; using Rdmp.UI.ItemActivation; +using Rdmp.Core.Curation.Data.Datasets; namespace Rdmp.UI.SubComponents; public partial class DatasetConfigurationUI : DatsetConfigurationUI_Design, IRefreshBusSubscriber From 9f7a04edbbce707b95e61eee55f8b074b550ca48 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 25 Apr 2025 09:23:13 +0100 Subject: [PATCH 002/142] update icons --- Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index 09025ba357..65d7f33850 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -784,13 +784,13 @@ ..\StandardRegex31.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - ..\AllDatasetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + ..\DataProvider.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 ..\AllDatasetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + \ No newline at end of file From 3adaffcb75750442aca5c676282ddc0de208beaf Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 25 Apr 2025 10:11:22 +0100 Subject: [PATCH 003/142] fix tests --- .../CommandExecution/ExecuteCommandDeleteDatasetTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs index 155e2442e0..8b09efdb32 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs @@ -24,7 +24,7 @@ public void TestDeleteExistingDataset() public void TestDeleteNonExistantDataset() { Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); - var founddataset = new Core.Curation.Data.Dataset(); + var founddataset = new Core.Curation.Data.Datasets.Dataset(); var delCmd = new ExecuteCommandDeleteDataset(GetMockActivator(), founddataset); Assert.Throws(() => delCmd.Execute()); Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); From e7dbbf34ee52bedcf084bf7cf47d5a10a7d3973a Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 25 Apr 2025 11:02:52 +0100 Subject: [PATCH 004/142] fix patch --- .../Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql index a9fb77d75e..cb2ab64cbf 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql @@ -39,6 +39,7 @@ BEGIN ALTER TABLE [dbo].[Dataset] ADD [Type] [varchar](256) NULL, [Url] [varchar](256) NULL, + [Provider_ID] [int] NULL, FOREIGN KEY (Provider_ID) REFERENCES DatasetProviderConfiguration(ID) END GO \ No newline at end of file From eb38289c0410883c2d24569b754d5c1e61e8ff21 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 25 Apr 2025 11:18:23 +0100 Subject: [PATCH 005/142] tidy up --- Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs | 2 +- Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs | 2 +- Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs | 2 +- Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs | 4 +--- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs index f420575814..b17dbecde6 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace Rdmp.Core.Datasets; +namespace Rdmp.Core.Curation.Data.Datasets; /// /// Base interface for dataset providers to impliment /// diff --git a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs index f3641d86ff..b346b35d63 100644 --- a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Linq; -namespace Rdmp.Core.Datasets; +namespace Rdmp.Core.Curation.Data.Datasets; /// /// Provider for internal datasets diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs index 11eb14104a..ab5877b396 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs @@ -2,7 +2,7 @@ using System.Data.Common; -namespace Rdmp.Core.Datasets +namespace Rdmp.Core.Curation.Data.Datasets { /// /// Base class to allow all plugin dataset types to be based off diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 958126e86e..89197604bd 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -1,6 +1,4 @@ using Rdmp.Core.CommandExecution; -using Rdmp.Core.Curation.Data; -using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Repositories; using System; using System.Collections.Generic; @@ -8,7 +6,7 @@ using System.Text; using System.Threading.Tasks; -namespace Rdmp.Core.Datasets; +namespace Rdmp.Core.Curation.Data.Datasets; public abstract class PluginDatasetProvider : IDatasetProvider { From ee442b63a166431ee7eae0ea1ea6e87096be8830 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 25 Apr 2025 12:50:19 +0100 Subject: [PATCH 006/142] add documentation --- Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs b/Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs index f255cc6c2f..6db9794192 100644 --- a/Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs +++ b/Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs @@ -11,6 +11,9 @@ namespace Rdmp.Core.Curation.Data.Datasets { + /// + /// Linkage between a and a . This is used to link datasets to catalogues in the RDMP system. + /// public class CatalogueDatasetLinkage: DatabaseEntity { private int _catalogueID; From ddca36aaffb45b368dc91465d7cd9f8acddc7788 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 25 Apr 2025 13:04:54 +0100 Subject: [PATCH 007/142] fix up tests --- .../IconProvision/CatalogueIconProvider.cs | 7 ++++++- .../IconProvision/CatalogueIcons.zh-Hans.resx | 6 +++--- Tests.Common/UnitTests.cs | 19 ++++++++++++++++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIconProvider.cs b/Rdmp.Core/Icons/IconProvision/CatalogueIconProvider.cs index f6db65f99a..a87b7cda33 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIconProvider.cs +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIconProvider.cs @@ -14,6 +14,7 @@ using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort.Joinables; using Rdmp.Core.Curation.Data.Dashboarding; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.Icons.IconOverlays; using Rdmp.Core.Icons.IconProvision.StateBasedIconProviders; @@ -193,9 +194,13 @@ protected virtual Image GetImageImpl(object concept, OverlayKind kind = } case IAtomicCommand cmd: return cmd.GetImage(this); + case Curation.Data.Datasets.Dataset: + case Curation.Data.Datasets.DatasetProviderConfiguration: + return GetImageImpl(RDMPConcept.Dataset); + case CatalogueDatasetLinkage: + return GetImageImpl(RDMPConcept.Catalogue); } - return ImageUnknown; } diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx index a3c0205507..6d2ad758fb 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx @@ -121,13 +121,13 @@ ..\zh-Hans\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - + + + \ No newline at end of file diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index 2395d65f5f..fe1613e925 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -17,6 +17,7 @@ using FAnsi.Implementations.Oracle; using FAnsi.Implementations.PostgreSql; using NUnit.Framework; +using Org.BouncyCastle.Asn1.X509.Qualified; using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandLine.Interactive; using Rdmp.Core.Curation; @@ -27,6 +28,7 @@ using Rdmp.Core.Curation.Data.Cohort.Joinables; using Rdmp.Core.Curation.Data.Dashboarding; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Pipelines; @@ -604,7 +606,22 @@ public static T WhenIHaveA(MemoryDataExportRepository repository) where T : D { return (T)(object)new RegexRedactionKey(repository.CatalogueRepository,WhenIHaveA(repository),WhenIHaveA(repository),"PK"); } - + if(typeof(T) == typeof(CatalogueDatasetLinkage)) + { + return (T)(object)new CatalogueDatasetLinkage(repository.CatalogueRepository, WhenIHaveA(repository), WhenIHaveA(repository)); + } + if(typeof(T) == typeof(Dataset)) + { + return (T)(object)new Dataset(repository.CatalogueRepository, "Dataset"); + } + if (typeof(T) == typeof(PluginDataset)) + { + return (T)(object)new PluginDataset(repository.CatalogueRepository, "Plugin Dataset"); + } + if (typeof(T) == typeof(DatasetProviderConfiguration)) + { + return (T)(object)new DatasetProviderConfiguration(repository.CatalogueRepository, "","","",WhenIHaveA(repository).ID,""); + } throw new TestCaseNotWrittenYetException(typeof(T)); } From 5772a074221262d0938d657ab4f64212eef7c7bf Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 25 Apr 2025 13:59:13 +0100 Subject: [PATCH 008/142] update help text --- Rdmp.Core/Curation/KeywordHelp.txt | 6 +++++- .../CatalogueDatabase/up/091_AddDatasetUpdates.sql | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Curation/KeywordHelp.txt b/Rdmp.Core/Curation/KeywordHelp.txt index aa5a3742b6..ae633fd750 100644 --- a/Rdmp.Core/Curation/KeywordHelp.txt +++ b/Rdmp.Core/Curation/KeywordHelp.txt @@ -99,4 +99,8 @@ FK_Redaction_RedactionConfiguration_ID: Prevents you from deleting a redaction c FK_Redaction_ColumnInfo_ID: Prevents redactions from becoming orphaned from their associated catalogue columns FK_RedactionKey_Redaction_ID: Prevents redaction primary keys from being orphaned from the associated redaction FK_RedactionKey_ColumnInfo_ID: Prevents redaction keys from becoming orphaned from their associated catalogue columns -FK_LoadMetadataRootReference: Links versions of a load metadata back to the source load metadata \ No newline at end of file +FK_LoadMetadataRootReference: Links versions of a load metadata back to the source load metadata +FK_CatalogueDatasetLinkage_Catalogue_ID:Linkage to Catalogue +FK_CatalogueDatasetLinkage_Dataset_ID: Linkage ot Dataset +FK_DatasetProviderConfiguration_DataAccessCredentials: Linkage to DataAccess Credentials +FK_DatasetProviderDatasetProviderCredentials: Links dataset to dataset provider \ No newline at end of file diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql index cb2ab64cbf..33c4793cd8 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql @@ -40,6 +40,6 @@ ALTER TABLE [dbo].[Dataset] ADD [Type] [varchar](256) NULL, [Url] [varchar](256) NULL, [Provider_ID] [int] NULL, - FOREIGN KEY (Provider_ID) REFERENCES DatasetProviderConfiguration(ID) + Contraint FK_DatasetProviderDatasetProviderCredentials FOREIGN KEY (Provider_ID) REFERENCES DatasetProviderConfiguration(ID) END GO \ No newline at end of file From f0ed05ff8fb2f33013b80fd4c12660a68ac92fee Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 25 Apr 2025 14:07:04 +0100 Subject: [PATCH 009/142] fix patch --- .../Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql index 33c4793cd8..6d0e540ca1 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql @@ -40,6 +40,6 @@ ALTER TABLE [dbo].[Dataset] ADD [Type] [varchar](256) NULL, [Url] [varchar](256) NULL, [Provider_ID] [int] NULL, - Contraint FK_DatasetProviderDatasetProviderCredentials FOREIGN KEY (Provider_ID) REFERENCES DatasetProviderConfiguration(ID) + CONSTRAINT FK_DatasetProviderDatasetProviderCredentials FOREIGN KEY (Provider_ID) REFERENCES DatasetProviderConfiguration(ID) END GO \ No newline at end of file From c6ac67241788b553d79f65275caf035e8153efff Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 25 Apr 2025 14:23:16 +0100 Subject: [PATCH 010/142] remove constraint --- Rdmp.Core/Curation/KeywordHelp.txt | 1 - .../Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Rdmp.Core/Curation/KeywordHelp.txt b/Rdmp.Core/Curation/KeywordHelp.txt index ae633fd750..63cccdfb8b 100644 --- a/Rdmp.Core/Curation/KeywordHelp.txt +++ b/Rdmp.Core/Curation/KeywordHelp.txt @@ -103,4 +103,3 @@ FK_LoadMetadataRootReference: Links versions of a load metadata back to the sour FK_CatalogueDatasetLinkage_Catalogue_ID:Linkage to Catalogue FK_CatalogueDatasetLinkage_Dataset_ID: Linkage ot Dataset FK_DatasetProviderConfiguration_DataAccessCredentials: Linkage to DataAccess Credentials -FK_DatasetProviderDatasetProviderCredentials: Links dataset to dataset provider \ No newline at end of file diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql index 6d0e540ca1..016a74e0b3 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql @@ -39,7 +39,6 @@ BEGIN ALTER TABLE [dbo].[Dataset] ADD [Type] [varchar](256) NULL, [Url] [varchar](256) NULL, - [Provider_ID] [int] NULL, - CONSTRAINT FK_DatasetProviderDatasetProviderCredentials FOREIGN KEY (Provider_ID) REFERENCES DatasetProviderConfiguration(ID) + [Provider_ID] [int] NULL END GO \ No newline at end of file From 1196e88d2ef6a6ca21ba43d7f4db14626d3f2226 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 09:23:06 +0100 Subject: [PATCH 011/142] add missing command --- ...ngCataloguesIntoExternalDatasetProvider.cs | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs new file mode 100644 index 0000000000..021c632296 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -0,0 +1,64 @@ +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NPOI.OpenXmlFormats.Vml.Office; + +namespace Rdmp.Core.CommandExecution.AtomicCommands +{ + public class ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider : BasicCommandExecution, IAtomicCommand + { + + private readonly IBasicActivateItems _activator; + private readonly PluginDatasetProvider _provider; + private readonly bool _includeExtractable; + private readonly bool _includeInternal; + private readonly bool _includeProjectSpecific; + private readonly bool _includeDeprecated; + private readonly string _id; + + public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicActivateItems activator, PluginDatasetProvider provider, string id, bool includeExtractable, bool includeInternal, bool includeProjectSpecific, bool includeDeprecated) + { + _activator = activator; + _provider = provider; + _id = id; + _includeExtractable = includeExtractable; + _includeInternal = includeInternal; + _includeProjectSpecific = includeProjectSpecific; + _includeDeprecated = includeDeprecated; + } + + + public override void Execute() + { + var catalogues = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); + if (!_includeInternal) + { + catalogues = catalogues.Where(c => !c.IsInternalDataset).ToList(); + } + if (!_includeProjectSpecific) + { + catalogues = catalogues.Where(c => !c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)).ToList(); + } + if (!_includeDeprecated) + { + catalogues = catalogues.Where(c => !c.IsDeprecated).ToList(); + } + if (!_includeExtractable) + { + catalogues = catalogues.Where(c => !c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList(); + } + //todo check this catalogue filtering works + foreach (var catalogue in catalogues) + { + var dataset = _provider.Create(catalogue); + var ds = _provider.AddExistingDatasetWithReturn(null, _id); + var cmd = new ExecuteCommandLinkCatalogueToDataset(_activator, catalogue, ds); + cmd.Execute(); + } + } + } +} From c95d55664a6b626f9e28c92605e04b93425818f2 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 09:28:57 +0100 Subject: [PATCH 012/142] fix ids --- ...mportExistingCataloguesIntoExternalDatasetProvider.cs | 9 ++++----- Rdmp.Core/Curation/Data/Datasets/Dataset.cs | 5 +++++ Rdmp.Core/Curation/Data/Datasets/IDataset.cs | 2 ++ .../Curation/Data/Datasets/PluginDatasetProvider.cs | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs index 021c632296..c6f1cc7d5e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -20,11 +20,10 @@ public class ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider : private readonly bool _includeDeprecated; private readonly string _id; - public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicActivateItems activator, PluginDatasetProvider provider, string id, bool includeExtractable, bool includeInternal, bool includeProjectSpecific, bool includeDeprecated) + public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicActivateItems activator, PluginDatasetProvider provider, bool includeExtractable, bool includeInternal, bool includeProjectSpecific, bool includeDeprecated) { _activator = activator; _provider = provider; - _id = id; _includeExtractable = includeExtractable; _includeInternal = includeInternal; _includeProjectSpecific = includeProjectSpecific; @@ -55,9 +54,9 @@ public override void Execute() foreach (var catalogue in catalogues) { var dataset = _provider.Create(catalogue); - var ds = _provider.AddExistingDatasetWithReturn(null, _id); - var cmd = new ExecuteCommandLinkCatalogueToDataset(_activator, catalogue, ds); - cmd.Execute(); + var ds = _provider.AddExistingDatasetWithReturn(null, dataset.GetID()); + var cmd = new ExecuteCommandLinkCatalogueToDataset(_activator, catalogue, ds); + cmd.Execute(); } } } diff --git a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs index 706de02b38..8c8ad8042e 100644 --- a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs @@ -75,6 +75,11 @@ public int? Provider_ID } public override string ToString() => Name; + public string GetID() + { + return ID.ToString(); + } + public List GetLinkedCatalogues() { return CatalogueRepository.GetAllObjectsWhere("Dataset_ID", this.ID).Select(l => l.Catalogue).Distinct().ToList(); diff --git a/Rdmp.Core/Curation/Data/Datasets/IDataset.cs b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs index c0b91946b6..f253db4c0f 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs @@ -39,4 +39,6 @@ public interface IDataset: IMapsDirectlyToDatabaseTable /// The URL to access the dataset /// string Url { get; } + + public abstract string GetID(); } diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 89197604bd..537069e8e7 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -29,7 +29,7 @@ protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderCo public abstract Curation.Data.Datasets.Dataset AddExistingDatasetWithReturn(string name, string url); - public abstract Curation.Data.Datasets.Dataset Create(Catalogue catalogue); + public abstract Curation.Data.Datasets.IDataset Create(Catalogue catalogue); public abstract void Update(string uuid, PluginDataset datasetUpdates); public abstract void UpdateUsingCatalogue(PluginDataset dataset, Catalogue catalogue); From 938e0a0c2da0737ea261b74526dc33c9ddb0a738 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 09:31:27 +0100 Subject: [PATCH 013/142] fix build --- ...CommandImportExistingCataloguesIntoExternalDatasetProvider.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs index c6f1cc7d5e..538bee1153 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -18,7 +18,6 @@ public class ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider : private readonly bool _includeInternal; private readonly bool _includeProjectSpecific; private readonly bool _includeDeprecated; - private readonly string _id; public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicActivateItems activator, PluginDatasetProvider provider, bool includeExtractable, bool includeInternal, bool includeProjectSpecific, bool includeDeprecated) { From fe1297acbcae0a1d7fc186d8abbc2c58a6675f38 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 09:43:36 +0100 Subject: [PATCH 014/142] fix tests --- Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs b/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs index d0463360d8..95b4a126aa 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs @@ -75,7 +75,8 @@ private List allowedToBeIncompatible typeof(ExecuteCommandSetArgument), typeof(ExecuteCommandAddToSession), typeof(ExecuteCommandDeletePlugin), - typeof(ExecuteCommandPerformRegexRedactionOnCatalogue) + typeof(ExecuteCommandPerformRegexRedactionOnCatalogue), + typeof(ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider) }); [Test] From 1ae5332b74c90dede384e183131dab4fb078b6f1 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 10:10:50 +0100 Subject: [PATCH 015/142] add provider edit --- ...henTargetIsDatasetProviderConfiguration.cs | 34 ++++ ...DatasetProviderConfigurationUI.Designer.cs | 173 ++++++++++++++++++ .../DatasetProviderConfigurationUI.cs | 78 ++++++++ 3 files changed, 285 insertions(+) create mode 100644 Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDatasetProviderConfiguration.cs create mode 100644 Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs create mode 100644 Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDatasetProviderConfiguration.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDatasetProviderConfiguration.cs new file mode 100644 index 0000000000..d0bbd45b38 --- /dev/null +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDatasetProviderConfiguration.cs @@ -0,0 +1,34 @@ +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.SubComponents; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.UI.CommandExecution.Proposals +{ + public class ProposeExecutionWhenTargetIsDatasetProviderConfiguration : RDMPCommandExecutionProposal + { + public ProposeExecutionWhenTargetIsDatasetProviderConfiguration(IActivateItems itemActivator) : base(itemActivator) + { + } + + public override void Activate(DatasetProviderConfiguration target) + { + ItemActivator.Activate(target); + } + + public override bool CanActivate(DatasetProviderConfiguration target) + { + return true; + } + + public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, DatasetProviderConfiguration target, InsertOption insertOption = InsertOption.Default) + { + return null; + } + } +} diff --git a/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs new file mode 100644 index 0000000000..79b828a1f1 --- /dev/null +++ b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs @@ -0,0 +1,173 @@ +namespace Rdmp.UI.SubComponents +{ + partial class DatasetProviderConfigurationUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Component Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + tbName = new System.Windows.Forms.TextBox(); + btnSave = new System.Windows.Forms.Button(); + tbType = new System.Windows.Forms.TextBox(); + label2 = new System.Windows.Forms.Label(); + tbUrl = new System.Windows.Forms.TextBox(); + label3 = new System.Windows.Forms.Label(); + tbOrgId = new System.Windows.Forms.TextBox(); + label4 = new System.Windows.Forms.Label(); + cbAccessCredentials = new System.Windows.Forms.ComboBox(); + label5 = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(102, 75); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(42, 15); + label1.TabIndex = 0; + label1.Text = "Name:"; + label1.Click += label1_Click; + // + // tbName + // + tbName.Location = new System.Drawing.Point(146, 72); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(252, 23); + tbName.TabIndex = 1; + // + // btnSave + // + btnSave.Location = new System.Drawing.Point(323, 279); + btnSave.Name = "btnSave"; + btnSave.Size = new System.Drawing.Size(75, 23); + btnSave.TabIndex = 2; + btnSave.Text = "Save"; + btnSave.UseVisualStyleBackColor = true; + btnSave.Click += btnSave_Click; + // + // tbType + // + tbType.Enabled = false; + tbType.Location = new System.Drawing.Point(146, 115); + tbType.Name = "tbType"; + tbType.Size = new System.Drawing.Size(252, 23); + tbType.TabIndex = 4; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(102, 118); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(34, 15); + label2.TabIndex = 3; + label2.Text = "Type:"; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(146, 154); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(252, 23); + tbUrl.TabIndex = 6; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(102, 157); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(31, 15); + label3.TabIndex = 5; + label3.Text = "URL:"; + label3.Click += label3_Click; + // + // tbOrgId + // + tbOrgId.Location = new System.Drawing.Point(146, 194); + tbOrgId.Name = "tbOrgId"; + tbOrgId.Size = new System.Drawing.Size(252, 23); + tbOrgId.TabIndex = 8; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(47, 197); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(92, 15); + label4.TabIndex = 7; + label4.Text = "Organisation ID:"; + // + // cbAccessCredentials + // + cbAccessCredentials.FormattingEnabled = true; + cbAccessCredentials.Location = new System.Drawing.Point(146, 236); + cbAccessCredentials.Name = "cbAccessCredentials"; + cbAccessCredentials.Size = new System.Drawing.Size(252, 23); + cbAccessCredentials.TabIndex = 9; + // + // label5 + // + label5.AutoSize = true; + label5.Location = new System.Drawing.Point(1, 239); + label5.Name = "label5"; + label5.Size = new System.Drawing.Size(135, 15); + label5.TabIndex = 10; + label5.Text = "Data Access Credentials:"; + // + // DatasetProviderConfigurationUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + Controls.Add(label5); + Controls.Add(cbAccessCredentials); + Controls.Add(tbOrgId); + Controls.Add(label4); + Controls.Add(tbUrl); + Controls.Add(label3); + Controls.Add(tbType); + Controls.Add(label2); + Controls.Add(btnSave); + Controls.Add(tbName); + Controls.Add(label1); + Name = "DatasetProviderConfigurationUI"; + Size = new System.Drawing.Size(1003, 530); + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private System.Windows.Forms.TextBox tbName; + private System.Windows.Forms.Button btnSave; + private System.Windows.Forms.TextBox tbType; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.TextBox tbOrgId; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.ComboBox cbAccessCredentials; + private System.Windows.Forms.Label label5; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs new file mode 100644 index 0000000000..f5aacbf2c1 --- /dev/null +++ b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs @@ -0,0 +1,78 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.Refreshing; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SubComponents; + +public partial class DatasetProviderConfigurationUI : DatasetProviderConfigurationUI_Design, IRefreshBusSubscriber +{ + private DatasetProviderConfiguration _configuration; + private IActivateItems _activator; + public DatasetProviderConfigurationUI() + { + InitializeComponent(); + } + + public override void SetDatabaseObject(IActivateItems activator, DatasetProviderConfiguration databaseObject) + { + _activator = activator; + base.SetDatabaseObject(activator, databaseObject); + _configuration = databaseObject; + tbName.Text = _configuration.Name; + tbOrgId.Text = _configuration.Organisation_ID; + tbType.Text = _configuration.Type; + //if (_configuration.Type == typeof(JiraDatasetProvider).ToString()) + //{ + // label3.Text = "Object Schema:"; + // label3.Location = new Point(label3.Location.X - 50, label3.Location.Y); + //} + tbUrl.Text = _configuration.Url; + cbAccessCredentials.Items.Clear(); + var credentials = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToArray(); + cbAccessCredentials.Items.AddRange(credentials); + cbAccessCredentials.SelectedIndex = Array.FindIndex(credentials, c => c.ID == _configuration.DataAccessCredentials_ID); + } + + private void label1_Click(object sender, EventArgs e) + { + + } + + private void label3_Click(object sender, EventArgs e) + { + + } + + private void btnSave_Click(object sender, EventArgs e) + { + _configuration.Name = tbName.Text; + _configuration.Url = tbUrl.Text; + _configuration.DataAccessCredentials_ID = ((DataAccessCredentials)cbAccessCredentials.SelectedItem).ID; + _configuration.Organisation_ID = tbOrgId.Text; + _configuration.SaveToDatabase(); + Publish(_configuration); + _activator.Show("Updated Dataset Provider Configuration"); + } + + public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) + { + } +} + +[TypeDescriptionProvider( + typeof(AbstractControlDescriptionProvider))] +public abstract class + DatasetProviderConfigurationUI_Design : RDMPSingleDatabaseObjectControl +{ +} \ No newline at end of file From dfc492c1b752b2146b211ccd12e434e2c28a3ef3 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 13:04:54 +0100 Subject: [PATCH 016/142] add linkage --- .../ExecuteCommandLinkCatalogueToDataset.cs | 19 +-- Rdmp.Core/Curation/Data/Catalogue.cs | 23 +++ Rdmp.Core/Curation/Data/Datasets/Dataset.cs | 6 +- .../Datasets/DatasetProviderConfiguration.cs | 14 ++ Rdmp.Core/Curation/Data/Datasets/IDataset.cs | 1 + .../Data/Datasets/IDatasetProvider.cs | 5 +- .../Data/Datasets/InternalDatasetProvider.cs | 15 ++ .../Data/Datasets/PluginDatasetProvider.cs | 9 ++ .../MainFormUITabs/CatalogueUI.Designer.cs | 140 ++++++++++++++++++ Rdmp.UI/MainFormUITabs/CatalogueUI.cs | 83 +++++++++++ .../CreateExternalDatasetDialog.Designer.cs | 83 +++++++++++ .../CreateExternalDatasetDialog.cs | 47 ++++++ .../LinkDatasetDialog.Designer.cs | 108 ++++++++++++++ Rdmp.UI/SubComponents/LinkDatasetDialog.cs | 54 +++++++ 14 files changed, 590 insertions(+), 17 deletions(-) create mode 100644 Rdmp.UI/SubComponents/CreateExternalDatasetDialog.Designer.cs create mode 100644 Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs create mode 100644 Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs create mode 100644 Rdmp.UI/SubComponents/LinkDatasetDialog.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs index 4e74fae767..d5b571c872 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs @@ -7,6 +7,7 @@ using System; using Rdmp.Core.Curation.Data; using System.Linq; +using Rdmp.Core.Curation.Data.Datasets; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -29,21 +30,9 @@ public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [Dema public override void Execute() { base.Execute(); - var items = _catalogue.CatalogueItems.ToList(); - foreach (var ci in items.Select(static item => item.ColumnInfo).Where(ci => ci?.Dataset_ID != _dataset.ID)) - { - ci.Dataset_ID = _dataset.ID; - ci.SaveToDatabase(); - if (!_linkAll) continue; - - var databaseName = ci.Name[..ci.Name.LastIndexOf('.')]; - var catalogueItems = ci.CatalogueRepository.GetAllObjects().Where(ci => ci.Name[..ci.Name.LastIndexOf(".", StringComparison.Ordinal)] == databaseName).ToList(); - foreach (var aci in catalogueItems) - { - aci.Dataset_ID = _dataset.ID; - aci.SaveToDatabase(); - } - } + var linkage = new CatalogueDatasetLinkage(_catalogue.CatalogueRepository, _catalogue, _dataset); + linkage.SaveToDatabase(); + Publish(linkage); } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 804c87f6f2..0a667abf55 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -14,8 +14,10 @@ using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; using Rdmp.Core.CohortCreation.Execution; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Curation.Data.Defaults; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Serialization; @@ -30,6 +32,7 @@ using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.DataAccess; +using Rdmp.Core.Startup; using Rdmp.Core.Ticketing; namespace Rdmp.Core.Curation.Data; @@ -1147,6 +1150,26 @@ internal Catalogue(ShareManager shareManager, ShareDefinition shareDefinition) /// public override string ToString() => Name; + public List GetLinkedDatasets() + { + return CatalogueRepository.GetAllObjectsWhere("Catalogue_ID", this.ID).Select(l => l.Dataset).Distinct().ToList(); + } + + public override void SaveToDatabase() + { + base.SaveToDatabase(); + foreach (var dataset in CatalogueRepository.GetAllObjectsWhere("Catalogue_ID", this.ID).Where(cdl => cdl.Autoupdate).Select(cld => cld.Dataset)) + { + var provider = CatalogueRepository.GetObjectByID((int)dataset.Provider_ID); + var providerConfiguration = CatalogueRepository.GetObjectByID((int)dataset.Provider_ID); + var repositoryProvider = new UserSettingsRepositoryFinder(); + var activator = new ThrowImmediatelyActivator(repositoryProvider, ThrowImmediatelyCheckNotifier.Quiet); + var providerInstance = providerConfiguration.GetProviderInstance(); + var ds = providerInstance.FetchDatasetByID(int.Parse(dataset.Url.Split('/').Last())); + providerInstance.UpdateUsingCatalogue(ds, this); + } + } + /// /// Sorts alphabetically based on /// diff --git a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs index 8c8ad8042e..971a098eac 100644 --- a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs @@ -17,7 +17,7 @@ namespace Rdmp.Core.Curation.Data.Datasets; /// -public class Dataset : DatabaseEntity, IDataset, IHasFolder +public class Dataset : DatabaseEntity, IDataset, IHasFolder { private string _name; private string _digitalObjectIdentifier; @@ -79,6 +79,10 @@ public string GetID() { return ID.ToString(); } + public string GetURL() + { + return Url.ToString(); + } public List GetLinkedCatalogues() { diff --git a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs index 125aaeb3c5..5ab1142800 100644 --- a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs @@ -44,6 +44,20 @@ internal DatasetProviderConfiguration(ICatalogueRepository repository, DbDataRea } + public IDatasetProvider GetProviderInstance() + { + var assemblies = AppDomain.CurrentDomain.GetAssemblies(); + foreach (var assembly in assemblies) + { + var type = assembly.GetTypes().FirstOrDefault(t => t.FullName == Type); + if (type != null) + { + return (IDatasetProvider)Activator.CreateInstance(type); + } + } + return null; + } + public override string ToString() { return _name; diff --git a/Rdmp.Core/Curation/Data/Datasets/IDataset.cs b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs index f253db4c0f..153b92e6e3 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs @@ -41,4 +41,5 @@ public interface IDataset: IMapsDirectlyToDatabaseTable string Url { get; } public abstract string GetID(); + public abstract string GetURL(); } diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs index b17dbecde6..2b623e9bf3 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs @@ -19,6 +19,9 @@ public interface IDatasetProvider /// /// /// - Curation.Data.Datasets.Dataset FetchDatasetByID(int id); + Dataset FetchDatasetByID(int id); + void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue); + Dataset Create(Catalogue catalogue); + Dataset AddExistingDatasetWithReturn(string name, string url); } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs index b346b35d63..6071edd409 100644 --- a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs @@ -15,6 +15,16 @@ public InternalDatasetProvider(IBasicActivateItems activator) _activator = activator; } + public Dataset AddExistingDatasetWithReturn(string name, string url) + { + throw new System.NotImplementedException(); + } + + public Dataset Create(Catalogue catalogue) + { + throw new System.NotImplementedException(); + } + /// public Curation.Data.Datasets.Dataset FetchDatasetByID(int id) { @@ -26,4 +36,9 @@ public Curation.Data.Datasets.Dataset FetchDatasetByID(int id) { return _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); } + + public void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) + { + throw new System.NotImplementedException(); + } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 537069e8e7..700c09404a 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -34,4 +34,13 @@ protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderCo public abstract void Update(string uuid, PluginDataset datasetUpdates); public abstract void UpdateUsingCatalogue(PluginDataset dataset, Catalogue catalogue); + public void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) + { + throw new NotImplementedException(); + } + + Dataset IDatasetProvider.Create(Catalogue catalogue) + { + throw new NotImplementedException(); + } } \ No newline at end of file diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs index 9ce5b6b790..ff2a867742 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs @@ -127,6 +127,16 @@ private void InitializeComponent() groupBox25 = new GroupBox(); aiAssociatedMedia = new SimpleControls.AdditionalInfomationUI(); ffAssociatedMedia = new SimpleControls.MultiSelectChips.FreeFormTextChipDisplay(); + tabPage8 = new TabPage(); + tableLayoutPanel3 = new TableLayoutPanel(); + label6 = new Label(); + label4 = new Label(); + label3 = new Label(); + label1 = new Label(); + label2 = new Label(); + label5 = new Label(); + button2 = new Button(); + button1 = new Button(); ((System.ComponentModel.ISupportInitialize)splitContainer1).BeginInit(); splitContainer1.Panel1.SuspendLayout(); splitContainer1.Panel2.SuspendLayout(); @@ -166,6 +176,8 @@ private void InitializeComponent() groupBox4.SuspendLayout(); tabPage7.SuspendLayout(); groupBox25.SuspendLayout(); + tabPage8.SuspendLayout(); + tableLayoutPanel3.SuspendLayout(); SuspendLayout(); // // splitContainer1 @@ -255,6 +267,8 @@ private void InitializeComponent() tabControl1.Controls.Add(tabPage5); tabControl1.Controls.Add(tabPage6); tabControl1.Controls.Add(tabPage7); + tabControl1.Controls.Add(tabPage8); + tabControl1.Dock = DockStyle.Fill; tabControl1.Location = new System.Drawing.Point(0, 0); tabControl1.Name = "tabControl1"; @@ -1100,6 +1114,118 @@ private void InitializeComponent() ffAssociatedMedia.Size = new System.Drawing.Size(300, 189); ffAssociatedMedia.TabIndex = 0; // + // tabPage8 + // + tabPage8.BackColor = System.Drawing.Color.WhiteSmoke; + tabPage8.Controls.Add(button1); + tabPage8.Controls.Add(tableLayoutPanel3); + tabPage8.Controls.Add(button2); + tabPage8.Location = new System.Drawing.Point(4, 24); + tabPage8.Name = "tabPage8"; + tabPage8.Size = new System.Drawing.Size(873, 895); + tabPage8.TabIndex = 7; + tabPage8.Text = "Datasets"; + // + // tableLayoutPanel3 + // + tableLayoutPanel3.AutoSize = true; + tableLayoutPanel3.CellBorderStyle = TableLayoutPanelCellBorderStyle.Single; + tableLayoutPanel3.ColumnCount = 6; + tableLayoutPanel3.ColumnStyles.Add(new ColumnStyle()); + tableLayoutPanel3.ColumnStyles.Add(new ColumnStyle()); + tableLayoutPanel3.ColumnStyles.Add(new ColumnStyle()); + tableLayoutPanel3.ColumnStyles.Add(new ColumnStyle()); + tableLayoutPanel3.ColumnStyles.Add(new ColumnStyle()); + tableLayoutPanel3.ColumnStyles.Add(new ColumnStyle()); + tableLayoutPanel3.Controls.Add(label6, 5, 0); + tableLayoutPanel3.Controls.Add(label4, 3, 0); + tableLayoutPanel3.Controls.Add(label3, 2, 0); + tableLayoutPanel3.Controls.Add(label1, 0, 0); + tableLayoutPanel3.Controls.Add(label2, 1, 0); + tableLayoutPanel3.Controls.Add(label5, 4, 0); + tableLayoutPanel3.Location = new System.Drawing.Point(7, 41); + tableLayoutPanel3.Name = "tableLayoutPanel3"; + tableLayoutPanel3.RowCount = 2; + tableLayoutPanel3.RowStyles.Add(new RowStyle(SizeType.Percent, 50F)); + tableLayoutPanel3.RowStyles.Add(new RowStyle(SizeType.Percent, 50F)); + tableLayoutPanel3.Size = new System.Drawing.Size(619, 100); + tableLayoutPanel3.TabIndex = 2; + tableLayoutPanel3.Paint += tableLayoutPanel3_Paint; + // + // label6 + // + label6.AutoSize = true; + label6.Location = new System.Drawing.Point(346, 1); + label6.Name = "label6"; + label6.Size = new System.Drawing.Size(51, 15); + label6.TabIndex = 7; + label6.Text = "Provider"; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(225, 1); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(32, 15); + label4.TabIndex = 3; + label4.Text = "View"; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(173, 1); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(45, 15); + label3.TabIndex = 5; + label3.Text = "Update"; + label3.Click += label3_Click; + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(4, 1); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(81, 15); + label1.TabIndex = 3; + label1.Text = "Dataset Name"; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(92, 1); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(74, 15); + label2.TabIndex = 4; + label2.Text = "Auto Update"; + // + // label5 + // + label5.AutoSize = true; + label5.Location = new System.Drawing.Point(264, 1); + label5.Name = "label5"; + label5.Size = new System.Drawing.Size(75, 15); + label5.TabIndex = 6; + label5.Text = "Remove Link"; + // + // button2 + // + button2.Location = new System.Drawing.Point(7, 12); + button2.Name = "button2"; + button2.Size = new System.Drawing.Size(150, 23); + button2.TabIndex = 1; + button2.Text = "Link to Existing Dataset"; + button2.UseVisualStyleBackColor = true; + button2.Click += button2_Click; + // + // button1 + // + button1.Location = new System.Drawing.Point(163, 12); + button1.Name = "button1"; + button1.Size = new System.Drawing.Size(183, 23); + button1.TabIndex = 3; + button1.Text = "Create using External Provider"; + button1.UseVisualStyleBackColor = true; + button1.Click += button1_Click; // CatalogueUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); @@ -1169,6 +1295,10 @@ private void InitializeComponent() tabPage7.ResumeLayout(false); groupBox25.ResumeLayout(false); groupBox25.PerformLayout(); + tabPage8.ResumeLayout(false); + tabPage8.PerformLayout(); + tableLayoutPanel3.ResumeLayout(false); + tableLayoutPanel3.PerformLayout(); ResumeLayout(false); } @@ -1268,5 +1398,15 @@ private void InitializeComponent() private SimpleControls.AdditionalInfomationUI aiAssociatedMedia; private SimpleControls.MultiSelectChips.DropdownOptionsChipDisplay ddDataSourceSetting; private SimpleControls.MultiSelectChips.DropdownOptionsChipDisplay ddDataSource; + private TabPage tabPage8; + private TableLayoutPanel tableLayoutPanel3; + private Button button2; + private Label label3; + private Label label1; + private Label label2; + private Label label4; + private Label label5; + private Label label6; + private Button button1; } } diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs index 46146327c1..ac21a86247 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs @@ -15,11 +15,13 @@ using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.ReusableLibraryCode; using Rdmp.UI.ItemActivation; using Rdmp.UI.Rules; using Rdmp.UI.ScintillaHelper; using Rdmp.UI.SimpleControls; using Rdmp.UI.SimpleDialogs; +using Rdmp.UI.SubComponents; using Rdmp.UI.TestsAndSetup.ServicePropogation; using ScintillaNET; @@ -452,7 +454,57 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) Bind(ffAssociatedMedia, "Value", "AssociatedMedia", c => c.AssociatedMedia); aiAssociatedMedia.TooltipText = CatalogueUIHelperText.AssociatedMedia; aiAssociatedMedia.SetItemActivator(Activator); + break; + case 7: + var datasets = _catalogue.GetLinkedDatasets(); + for (int i = 1; i < tableLayoutPanel3.RowCount; i++) + { + tableLayoutPanel3.RowStyles.RemoveAt(i); + } + foreach (var dataset in datasets) + { + var linkage = _catalogue.CatalogueRepository.GetAllObjectsWhere("Dataset_ID", dataset.ID).First(l => l.Catalogue.ID == _catalogue.ID); + tableLayoutPanel3.RowCount++; + var label = new Label(); + label.Text = dataset.Name; + label.AutoSize = true; + tableLayoutPanel3.Controls.Add(label, 0, tableLayoutPanel3.RowCount - 1); + var cb = new CheckBox(); + cb.Checked = linkage.Autoupdate; + cb.CheckedChanged += (object sender, EventArgs e) => + { + linkage.Autoupdate = !linkage.Autoupdate; + linkage.SaveToDatabase(); + Publish(linkage); + }; + tableLayoutPanel3.Controls.Add(cb, 1, tableLayoutPanel3.RowCount - 1); + var btn = new Button(); + btn.Text = "Update Now"; + btn.Click += (object sender, EventArgs e) => UpdateDataset(dataset); + tableLayoutPanel3.Controls.Add(btn, 2, tableLayoutPanel3.RowCount - 1); + //if (dataset.Type != typeof(JiraDatasetProvider).ToString()) + //{ + // var btn2 = new Button(); + // btn2.Text = "View"; + // btn2.Click += (object sender, EventArgs e) => UsefulStuff.OpenUrl(dataset.Url); + // tableLayoutPanel3.Controls.Add(btn2, 3, tableLayoutPanel3.RowCount - 1); + //} + var btn3 = new Button(); + btn3.Text = "Remove Link to Dataset"; + btn3.Click += (object sender, EventArgs e) => + { + if (Activator.YesNo("Are you sure?", "Remove Linkage")) + { + linkage.DeleteInDatabase(); + } + }; + tableLayoutPanel3.Controls.Add(btn3, 4, tableLayoutPanel3.RowCount - 1); + var labelType = new Label(); + labelType.Text = _catalogue.CatalogueRepository.GetObjectByID((int)dataset.Provider_ID).Name; + labelType.AutoSize = true; + tableLayoutPanel3.Controls.Add(labelType, 5, tableLayoutPanel3.RowCount - 1); + } break; default: break; @@ -460,6 +512,37 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) setTabBindings.Add(selectedIndex); } + private void UpdateDataset(Dataset dataset) + { + var providerConfiguration = Activator.RepositoryLocator.CatalogueRepository.GetObjectByID((int)dataset.Provider_ID); + var provider = providerConfiguration.GetProviderInstance(); + var ds = provider.FetchDatasetByID(int.Parse(dataset.Url.Split('/').Last())); //todo this id may change + provider.UpdateUsingCatalogue(ds, _catalogue); + } + private void tableLayoutPanel3_Paint(object sender, PaintEventArgs e) + { + + } + + private void label3_Click(object sender, EventArgs e) + { + + } + + private void button2_Click(object sender, EventArgs e) + { + var dialog = new LinkDatasetDialog(); + dialog.Setup(Activator, _catalogue); + dialog.Show(); + } + + private void button1_Click(object sender, EventArgs e) + { + var dialog = new CreateExternalDatasetDialog(); + dialog.Setup(Activator, _catalogue); + dialog.Show(); + } + private void btnStartDateClear_Click(object sender, EventArgs e) { dtpStart.CustomFormat = " "; diff --git a/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.Designer.cs b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.Designer.cs new file mode 100644 index 0000000000..c19d29eedb --- /dev/null +++ b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.Designer.cs @@ -0,0 +1,83 @@ +namespace Rdmp.UI.SubComponents +{ + partial class CreateExternalDatasetDialog + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + groupBox3 = new System.Windows.Forms.GroupBox(); + cbPovider = new System.Windows.Forms.ComboBox(); + button1 = new System.Windows.Forms.Button(); + groupBox3.SuspendLayout(); + SuspendLayout(); + // + // groupBox3 + // + groupBox3.Controls.Add(cbPovider); + groupBox3.Location = new System.Drawing.Point(12, 8); + groupBox3.Name = "groupBox3"; + groupBox3.Size = new System.Drawing.Size(200, 60); + groupBox3.TabIndex = 2; + groupBox3.TabStop = false; + groupBox3.Text = "Dataset Provider"; + // + // cbPovider + // + cbPovider.FormattingEnabled = true; + cbPovider.Location = new System.Drawing.Point(6, 22); + cbPovider.Name = "cbPovider"; + cbPovider.Size = new System.Drawing.Size(188, 23); + cbPovider.TabIndex = 1; + // + // button1 + // + button1.Location = new System.Drawing.Point(123, 74); + button1.Name = "button1"; + button1.Size = new System.Drawing.Size(89, 23); + button1.TabIndex = 3; + button1.Text = "Create"; + button1.UseVisualStyleBackColor = true; + button1.Click += button1_Click; + // + // CreateExternalDatasetDialog + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(303, 149); + Controls.Add(button1); + Controls.Add(groupBox3); + Name = "CreateExternalDatasetDialog"; + Text = "Create Dataset"; + groupBox3.ResumeLayout(false); + ResumeLayout(false); + } + + #endregion + private System.Windows.Forms.GroupBox groupBox3; + private System.Windows.Forms.ComboBox cbPovider; + private System.Windows.Forms.Button button1; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs new file mode 100644 index 0000000000..64745fe50d --- /dev/null +++ b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs @@ -0,0 +1,47 @@ +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data; +using Rdmp.UI.ItemActivation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SubComponents +{ + public partial class CreateExternalDatasetDialog : Form + { + IActivateItems _activator; + private Catalogue _catalogue; + + + public CreateExternalDatasetDialog() + { + InitializeComponent(); + } + + public void Setup(IActivateItems activator, Catalogue catalogue) + { + _activator = activator; + _catalogue = catalogue; + cbPovider.Items.AddRange(_activator.RepositoryLocator.CatalogueRepository.GetAllObjects()); + } + + private void button1_Click(object sender, EventArgs e) + { + var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(); + var dataset = provider.Create(_catalogue); + + if (!_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Dataset_ID", dataset.ID).Where(l => l.Catalogue.ID == _catalogue.ID).Any()) + { + var linkage = new CatalogueDatasetLinkage(_activator.RepositoryLocator.CatalogueRepository, _catalogue, dataset); + linkage.SaveToDatabase(); + } + Close(); + } + } +} diff --git a/Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs b/Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs new file mode 100644 index 0000000000..9a811a97f7 --- /dev/null +++ b/Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs @@ -0,0 +1,108 @@ +namespace Rdmp.UI.SubComponents +{ + partial class LinkDatasetDialog + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + groupBox2 = new System.Windows.Forms.GroupBox(); + tbID = new System.Windows.Forms.TextBox(); + groupBox3 = new System.Windows.Forms.GroupBox(); + cbPovider = new System.Windows.Forms.ComboBox(); + button1 = new System.Windows.Forms.Button(); + groupBox2.SuspendLayout(); + groupBox3.SuspendLayout(); + SuspendLayout(); + // + // groupBox2 + // + groupBox2.Controls.Add(tbID); + groupBox2.Location = new System.Drawing.Point(12, 82); + groupBox2.Name = "groupBox2"; + groupBox2.Size = new System.Drawing.Size(200, 60); + groupBox2.TabIndex = 2; + groupBox2.TabStop = false; + groupBox2.Text = "Dataset ID"; + // + // tbID + // + tbID.Location = new System.Drawing.Point(6, 20); + tbID.Name = "tbID"; + tbID.Size = new System.Drawing.Size(188, 23); + tbID.TabIndex = 0; + // + // groupBox3 + // + groupBox3.Controls.Add(cbPovider); + groupBox3.Location = new System.Drawing.Point(12, 8); + groupBox3.Name = "groupBox3"; + groupBox3.Size = new System.Drawing.Size(200, 60); + groupBox3.TabIndex = 2; + groupBox3.TabStop = false; + groupBox3.Text = "Dataset Provider"; + // + // cbPovider + // + cbPovider.FormattingEnabled = true; + cbPovider.Location = new System.Drawing.Point(6, 22); + cbPovider.Name = "cbPovider"; + cbPovider.Size = new System.Drawing.Size(188, 23); + cbPovider.TabIndex = 1; + // + // button1 + // + button1.Location = new System.Drawing.Point(216, 155); + button1.Name = "button1"; + button1.Size = new System.Drawing.Size(89, 23); + button1.TabIndex = 3; + button1.Text = "Link Dataset"; + button1.UseVisualStyleBackColor = true; + button1.Click += button1_Click; + // + // LinkDatasetDialog + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(800, 450); + Controls.Add(button1); + Controls.Add(groupBox3); + Controls.Add(groupBox2); + Name = "LinkDatasetDialog"; + Text = "Link Dataset"; + groupBox2.ResumeLayout(false); + groupBox2.PerformLayout(); + groupBox3.ResumeLayout(false); + ResumeLayout(false); + } + + #endregion + private System.Windows.Forms.GroupBox groupBox2; + private System.Windows.Forms.TextBox tbID; + private System.Windows.Forms.GroupBox groupBox3; + private System.Windows.Forms.ComboBox cbPovider; + private System.Windows.Forms.Button button1; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/LinkDatasetDialog.cs b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs new file mode 100644 index 0000000000..c8f0a5ab0f --- /dev/null +++ b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs @@ -0,0 +1,54 @@ +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data; +using Rdmp.UI.ItemActivation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SubComponents +{ + public partial class LinkDatasetDialog : Form + { + IActivateItems _activator; + private Catalogue _catalogue; + public LinkDatasetDialog() + { + InitializeComponent(); + } + + public void Setup(IActivateItems activator, Catalogue catalogue) + { + _activator = activator; + _catalogue = catalogue; + cbPovider.Items.AddRange(_activator.RepositoryLocator.CatalogueRepository.GetAllObjects()); + } + + private void button1_Click(object sender, EventArgs e) + { + var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(); + Dataset dataset; + var fetchedDataset = provider.FetchDatasetByID(int.Parse(tbID.Text));//todo is it always ints? + if (_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.GetURL()).Any()) + { + dataset = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.GetURL()).First(); + } + else + { + dataset = provider.AddExistingDatasetWithReturn(null, tbID.Text); + } + if (!_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Dataset_ID", dataset.ID).Where(l => l.Catalogue.ID == _catalogue.ID).Any()) + { + var linkage = new CatalogueDatasetLinkage(_activator.RepositoryLocator.CatalogueRepository, _catalogue, dataset); + linkage.SaveToDatabase(); + } + Close(); + + } + } +} From 1f36e5ffeecafd451fb13fa3df444efa3ab71877 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 13:22:56 +0100 Subject: [PATCH 017/142] fix instance create --- Rdmp.Core/Curation/Data/Catalogue.cs | 2 +- .../Curation/Data/Datasets/DatasetProviderConfiguration.cs | 5 +++-- Rdmp.UI/MainFormUITabs/CatalogueUI.cs | 2 +- Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs | 2 +- Rdmp.UI/SubComponents/LinkDatasetDialog.cs | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 0a667abf55..78ee80a344 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -1164,7 +1164,7 @@ public override void SaveToDatabase() var providerConfiguration = CatalogueRepository.GetObjectByID((int)dataset.Provider_ID); var repositoryProvider = new UserSettingsRepositoryFinder(); var activator = new ThrowImmediatelyActivator(repositoryProvider, ThrowImmediatelyCheckNotifier.Quiet); - var providerInstance = providerConfiguration.GetProviderInstance(); + var providerInstance = providerConfiguration.GetProviderInstance(activator); var ds = providerInstance.FetchDatasetByID(int.Parse(dataset.Url.Split('/').Last())); providerInstance.UpdateUsingCatalogue(ds, this); } diff --git a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs index 5ab1142800..1cf302fcfc 100644 --- a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs @@ -1,4 +1,5 @@ using MongoDB.Driver; +using Rdmp.Core.CommandExecution; using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; using Rdmp.Core.Repositories; using System; @@ -44,7 +45,7 @@ internal DatasetProviderConfiguration(ICatalogueRepository repository, DbDataRea } - public IDatasetProvider GetProviderInstance() + public IDatasetProvider GetProviderInstance(IBasicActivateItems activator) { var assemblies = AppDomain.CurrentDomain.GetAssemblies(); foreach (var assembly in assemblies) @@ -52,7 +53,7 @@ public IDatasetProvider GetProviderInstance() var type = assembly.GetTypes().FirstOrDefault(t => t.FullName == Type); if (type != null) { - return (IDatasetProvider)Activator.CreateInstance(type); + return (IDatasetProvider)Activator.CreateInstance(type, activator, this); } } return null; diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs index ac21a86247..53e0a1e639 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs @@ -515,7 +515,7 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) private void UpdateDataset(Dataset dataset) { var providerConfiguration = Activator.RepositoryLocator.CatalogueRepository.GetObjectByID((int)dataset.Provider_ID); - var provider = providerConfiguration.GetProviderInstance(); + var provider = providerConfiguration.GetProviderInstance(Activator); var ds = provider.FetchDatasetByID(int.Parse(dataset.Url.Split('/').Last())); //todo this id may change provider.UpdateUsingCatalogue(ds, _catalogue); } diff --git a/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs index 64745fe50d..9f4f59d492 100644 --- a/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs +++ b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs @@ -33,7 +33,7 @@ public void Setup(IActivateItems activator, Catalogue catalogue) private void button1_Click(object sender, EventArgs e) { - var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(); + var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(_activator); var dataset = provider.Create(_catalogue); if (!_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Dataset_ID", dataset.ID).Where(l => l.Catalogue.ID == _catalogue.ID).Any()) diff --git a/Rdmp.UI/SubComponents/LinkDatasetDialog.cs b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs index c8f0a5ab0f..f13d482309 100644 --- a/Rdmp.UI/SubComponents/LinkDatasetDialog.cs +++ b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs @@ -31,7 +31,7 @@ public void Setup(IActivateItems activator, Catalogue catalogue) private void button1_Click(object sender, EventArgs e) { - var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(); + var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(_activator); Dataset dataset; var fetchedDataset = provider.FetchDatasetByID(int.Parse(tbID.Text));//todo is it always ints? if (_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.GetURL()).Any()) From 582c7d63968b7794cf96859bba7dff023318e16a Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 13:31:27 +0100 Subject: [PATCH 018/142] add http clinet to provider --- .../Curation/Data/Datasets/DatasetProviderConfiguration.cs | 2 +- Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs | 3 ++- Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs index 1cf302fcfc..1e9d637fd0 100644 --- a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs @@ -53,7 +53,7 @@ public IDatasetProvider GetProviderInstance(IBasicActivateItems activator) var type = assembly.GetTypes().FirstOrDefault(t => t.FullName == Type); if (type != null) { - return (IDatasetProvider)Activator.CreateInstance(type, activator, this); + return (IDatasetProvider)Activator.CreateInstance(type, activator, this,null); } } return null; diff --git a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs index 6071edd409..78da56e6f9 100644 --- a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs @@ -1,6 +1,7 @@ using Rdmp.Core.CommandExecution; using System.Collections.Generic; using System.Linq; +using System.Net.Http; namespace Rdmp.Core.Curation.Data.Datasets; @@ -10,7 +11,7 @@ namespace Rdmp.Core.Curation.Data.Datasets; public class InternalDatasetProvider : IDatasetProvider { private readonly IBasicActivateItems _activator; - public InternalDatasetProvider(IBasicActivateItems activator) + public InternalDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration = null, HttpClient client = null) { _activator = activator; } diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 700c09404a..1af15271d5 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net.Http; using System.Text; using System.Threading.Tasks; @@ -14,7 +15,7 @@ public abstract class PluginDatasetProvider : IDatasetProvider protected ICatalogueRepository Repository { get; } protected IBasicActivateItems Activator { get; } - protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration) + protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration, HttpClient client = null) { Configuration = configuration; Activator = activator; From e912c6152da531564ad072157029737f51a25628 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 14:47:30 +0100 Subject: [PATCH 019/142] fix tests --- ...ecuteCommandLinkCatalogueToDatasetTests.cs | 21 ++----------------- Rdmp.Core/Curation/Data/Datasets/Dataset.cs | 4 ++-- .../Curation/Data/Datasets/PluginDataset.cs | 9 ++++++++ 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs index 330e1e3d46..55ce585c61 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs @@ -38,11 +38,6 @@ public void TestLinkCatalogueToDataset() var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static c => c.Name == "Dataset1"); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), foundCatalogue, founddataset); Assert.DoesNotThrow(linkCmd.Execute); - var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(); - foreach (var ci in columInfo) - { - Assert.That(ci.Dataset_ID, Is.EqualTo(founddataset.ID)); - } founddataset.DeleteInDatabase(); foundCatalogue.DeleteInDatabase(); @@ -76,18 +71,6 @@ public void TestLinkCatalogueToDatasetNotAll() var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(c => c.Name == "Dataset1"); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), foundCatalogue, founddataset, false); Assert.DoesNotThrow(linkCmd.Execute); - var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().Where(ci => _cata1.CatalogueItems.Contains(ci)); - foreach (var ci in columInfo) - { - Assert.That(ci.ColumnInfo.Dataset_ID, Is.EqualTo(founddataset.ID)); - } - - var columInfo2 = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().Where(ci => _cata2.CatalogueItems.Contains(ci)); - foreach (var ci in columInfo2) - { - Assert.That(ci.ColumnInfo.Dataset_ID, Is.Null); - } - } [Test] public void TestLinkCatalogueToDatasetBadCatalogue() @@ -105,11 +88,11 @@ public void TestLinkCatalogueToDatasetBadDataset() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); - var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository,"catalogue"), null, false); + var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "catalogue"), null, false); Assert.Throws(linkCmd.Execute); } - [Test] + [Test] public void TestLinkCatalogueToDatasetBadEverything() { var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), null, null, false); diff --git a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs index 971a098eac..25cca30266 100644 --- a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs @@ -75,11 +75,11 @@ public int? Provider_ID } public override string ToString() => Name; - public string GetID() + public virtual string GetID() { return ID.ToString(); } - public string GetURL() + public virtual string GetURL() { return Url.ToString(); } diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs index ab5877b396..06aa798179 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs @@ -13,5 +13,14 @@ public PluginDataset(ICatalogueRepository catalogueRepository, string name) : ba public PluginDataset() { } public PluginDataset(ICatalogueRepository repository, DbDataReader r) : base(repository, r) { } + + public override string GetID() + { + return ID.ToString(); + } + public override string GetURL() + { + return ID.ToString(); + } } } \ No newline at end of file From d7af0a7dc8480396ef0aa596404d8dd6f38ad5f8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 15:10:00 +0100 Subject: [PATCH 020/142] fix update --- Rdmp.Core/Curation/Data/Datasets/Dataset.cs | 4 ++-- Rdmp.Core/Curation/Data/Datasets/IDataset.cs | 2 +- Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs | 2 +- Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs | 6 +----- Rdmp.UI/MainFormUITabs/CatalogueUI.cs | 2 +- Rdmp.UI/SubComponents/LinkDatasetDialog.cs | 4 ++-- 6 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs index 25cca30266..f56d590b1d 100644 --- a/Rdmp.Core/Curation/Data/Datasets/Dataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs @@ -79,9 +79,9 @@ public virtual string GetID() { return ID.ToString(); } - public virtual string GetURL() + public virtual string GetRemoteID() { - return Url.ToString(); + return ID.ToString(); } public List GetLinkedCatalogues() diff --git a/Rdmp.Core/Curation/Data/Datasets/IDataset.cs b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs index 153b92e6e3..3b73096298 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs @@ -41,5 +41,5 @@ public interface IDataset: IMapsDirectlyToDatabaseTable string Url { get; } public abstract string GetID(); - public abstract string GetURL(); + public abstract string GetRemoteID(); } diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs index 06aa798179..1949d5c355 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs @@ -18,7 +18,7 @@ public override string GetID() { return ID.ToString(); } - public override string GetURL() + public override string GetRemoteID() { return ID.ToString(); } diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 1af15271d5..2a03e02a8d 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -33,12 +33,8 @@ protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderCo public abstract Curation.Data.Datasets.IDataset Create(Catalogue catalogue); public abstract void Update(string uuid, PluginDataset datasetUpdates); - public abstract void UpdateUsingCatalogue(PluginDataset dataset, Catalogue catalogue); - public void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) - { - throw new NotImplementedException(); - } + public abstract void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue); Dataset IDatasetProvider.Create(Catalogue catalogue) { diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs index 53e0a1e639..1ea6ee63b7 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs @@ -516,7 +516,7 @@ private void UpdateDataset(Dataset dataset) { var providerConfiguration = Activator.RepositoryLocator.CatalogueRepository.GetObjectByID((int)dataset.Provider_ID); var provider = providerConfiguration.GetProviderInstance(Activator); - var ds = provider.FetchDatasetByID(int.Parse(dataset.Url.Split('/').Last())); //todo this id may change + var ds = provider.FetchDatasetByID(int.Parse(dataset.GetRemoteID())); //todo this id may change provider.UpdateUsingCatalogue(ds, _catalogue); } private void tableLayoutPanel3_Paint(object sender, PaintEventArgs e) diff --git a/Rdmp.UI/SubComponents/LinkDatasetDialog.cs b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs index f13d482309..e7a98c5036 100644 --- a/Rdmp.UI/SubComponents/LinkDatasetDialog.cs +++ b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs @@ -34,9 +34,9 @@ private void button1_Click(object sender, EventArgs e) var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(_activator); Dataset dataset; var fetchedDataset = provider.FetchDatasetByID(int.Parse(tbID.Text));//todo is it always ints? - if (_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.GetURL()).Any()) + if (_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.Url).Any()) { - dataset = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.GetURL()).First(); + dataset = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.Url).First(); } else { From 853d2b78221019cb86ae179750703179a83b986a Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Apr 2025 15:46:57 +0100 Subject: [PATCH 021/142] fix update --- Rdmp.UI/MainFormUITabs/CatalogueUI.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs index 1ea6ee63b7..04ff867703 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs @@ -516,8 +516,7 @@ private void UpdateDataset(Dataset dataset) { var providerConfiguration = Activator.RepositoryLocator.CatalogueRepository.GetObjectByID((int)dataset.Provider_ID); var provider = providerConfiguration.GetProviderInstance(Activator); - var ds = provider.FetchDatasetByID(int.Parse(dataset.GetRemoteID())); //todo this id may change - provider.UpdateUsingCatalogue(ds, _catalogue); + provider.UpdateUsingCatalogue(dataset, _catalogue); } private void tableLayoutPanel3_Paint(object sender, PaintEventArgs e) { From 1669334fa58b7ad75a023fe9b82eae7f0ee1a778 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 29 Apr 2025 13:41:37 +0100 Subject: [PATCH 022/142] add generic dataset provider ui --- ...ngCataloguesIntoExternalDatasetProvider.cs | 4 +- .../Data/Datasets/IDatasetProvider.cs | 1 + .../Data/Datasets/InternalDatasetProvider.cs | 5 + .../Data/Datasets/PluginDatasetProvider.cs | 16 +- .../Collections/ConfigurationsCollectionUI.cs | 21 +- .../ExecuteCommandAddNewDatasetProviderUI.cs | 30 +++ .../ExecuteCommandAddNewDatasetUI.cs | 30 +++ ...DatasetProviderConfigurationUI.Designer.cs | 246 ++++++++++++++++++ ...CreateNewDatasetProviderConfigurationUI.cs | 81 ++++++ .../ImportExistingDatasetUI.Designer.cs | 131 ++++++++++ .../Datasets/ImportExistingDatasetUI.cs | 77 ++++++ 11 files changed, 632 insertions(+), 10 deletions(-) create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewDatasetProviderUI.cs create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewDatasetUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs index 538bee1153..5a52e9255b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -13,13 +13,13 @@ public class ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider : { private readonly IBasicActivateItems _activator; - private readonly PluginDatasetProvider _provider; + private readonly IDatasetProvider _provider; private readonly bool _includeExtractable; private readonly bool _includeInternal; private readonly bool _includeProjectSpecific; private readonly bool _includeDeprecated; - public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicActivateItems activator, PluginDatasetProvider provider, bool includeExtractable, bool includeInternal, bool includeProjectSpecific, bool includeDeprecated) + public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicActivateItems activator, IDatasetProvider provider, bool includeExtractable, bool includeInternal, bool includeProjectSpecific, bool includeDeprecated) { _activator = activator; _provider = provider; diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs index 2b623e9bf3..7ad62ce065 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs @@ -24,4 +24,5 @@ public interface IDatasetProvider Dataset Create(Catalogue catalogue); Dataset AddExistingDatasetWithReturn(string name, string url); + Dataset AddExistingDataset(string name, string url); } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs index 78da56e6f9..a61f66153d 100644 --- a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs @@ -16,6 +16,11 @@ public InternalDatasetProvider(IBasicActivateItems activator, DatasetProviderCon _activator = activator; } + public Dataset AddExistingDataset(string name, string url) + { + throw new System.NotImplementedException(); + } + public Dataset AddExistingDatasetWithReturn(string name, string url) { throw new System.NotImplementedException(); diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 2a03e02a8d..5e6c3f95be 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -22,21 +22,23 @@ protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderCo Repository = activator.RepositoryLocator.CatalogueRepository; } - public abstract Curation.Data.Datasets.Dataset FetchDatasetByID(int id); + public abstract Dataset FetchDatasetByID(int id); - public abstract List FetchDatasets(); + public abstract List FetchDatasets(); - public abstract void AddExistingDataset(string name, string url); + public abstract Dataset AddExistingDatasetWithReturn(string name, string url); - public abstract Curation.Data.Datasets.Dataset AddExistingDatasetWithReturn(string name, string url); - - public abstract Curation.Data.Datasets.IDataset Create(Catalogue catalogue); public abstract void Update(string uuid, PluginDataset datasetUpdates); public abstract void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue); - Dataset IDatasetProvider.Create(Catalogue catalogue) + public virtual Dataset Create(Catalogue catalogue) + { + throw new NotImplementedException(); + } + + public virtual Dataset AddExistingDataset(string name, string url) { throw new NotImplementedException(); } diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index c75dc69052..20dfe69886 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -7,6 +7,8 @@ using Rdmp.Core.Providers.Nodes; using Rdmp.Core.Curation.DataHelper.RegexRedaction; using Rdmp.Core.Curation.Data.Datasets; +using System; +using static Rdmp.Core.Curation.Data.Catalogue; namespace Rdmp.UI.Collections; @@ -22,7 +24,9 @@ public ConfigurationsCollectionUI() private IAtomicCommand[] GetWhitespaceRightClickMenu() { - return new IAtomicCommand[] + var datasetProviders = AppDomain.CurrentDomain.GetAssemblies().SelectMany(s => s.GetTypes()).Where(p => typeof(PluginDatasetProvider).IsAssignableFrom(p) && !p.IsAbstract); + + var options = new IAtomicCommand[] { new ExecuteCommandCreateNewDatasetUI(_activator){ OverrideCommandName="Add New Dataset" @@ -32,6 +36,21 @@ private IAtomicCommand[] GetWhitespaceRightClickMenu() OverrideCommandName="Add New Regex Redaction Configuration" } }; + foreach(var provider in datasetProviders) + { + options = options.Append(new ExecuteCommandAddNewDatasetProviderUI(_activator,provider) + { + OverrideCommandName = $"Add New {System.Text.RegularExpressions.Regex.Replace(provider.Name, "([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim()}", + SuggestedCategory = "Dataset Provider Configurations" + }).ToArray(); + options = options.Append(new ExecuteCommandAddNewDatasetUI(_activator, provider) + { + OverrideCommandName = $"Add Existing {System.Text.RegularExpressions.Regex.Replace(provider.Name, "([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim().Replace("Provider","")}", + SuggestedCategory = "Datasets" + }).ToArray(); + } + + return options; } public override void SetItemActivator(IActivateItems activator) diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewDatasetProviderUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewDatasetProviderUI.cs new file mode 100644 index 0000000000..5caf9d5fa4 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewDatasetProviderUI.cs @@ -0,0 +1,30 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.SimpleDialogs.Datasets; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.UI.CommandExecution.AtomicCommands +{ + class ExecuteCommandAddNewDatasetProviderUI: BasicUICommandExecution, IAtomicCommand + { + private readonly IActivateItems _activator; + private readonly Type _provider; + + public ExecuteCommandAddNewDatasetProviderUI(IActivateItems activator, Type provider) : base(activator) + { + _activator = activator; + _provider = provider; + } + + public override void Execute() + { + base.Execute(); + var ui = new CreateNewDatasetProviderConfigurationUI(_activator, _provider); + ui.ShowDialog(); + } + } +} diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewDatasetUI.cs new file mode 100644 index 0000000000..155b534625 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewDatasetUI.cs @@ -0,0 +1,30 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.SimpleDialogs.Datasets; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.UI.CommandExecution.AtomicCommands +{ + class ExecuteCommandAddNewDatasetUI : ExecuteCommandCreateDataset + { + private readonly IActivateItems _activator; + private readonly Type _providerType; + + public ExecuteCommandAddNewDatasetUI(IActivateItems activator, Type providerType) : base( + activator, "New Dataset") + { + _activator = activator; + _providerType = providerType; ; + } + + public override void Execute() + { + var ui = new ImportExistingDatasetUI(_activator, _providerType); + ui.ShowDialog(); + } + } +} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs new file mode 100644 index 0000000000..d87e84e69f --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs @@ -0,0 +1,246 @@ +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + partial class CreateNewDatasetProviderConfigurationUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + label2 = new System.Windows.Forms.Label(); + label3 = new System.Windows.Forms.Label(); + label4 = new System.Windows.Forms.Label(); + tbName = new System.Windows.Forms.TextBox(); + tbUrl = new System.Windows.Forms.TextBox(); + tbOrganisationId = new System.Windows.Forms.TextBox(); + cbCredentials = new System.Windows.Forms.ComboBox(); + btnSave = new System.Windows.Forms.Button(); + cbImportCatalogues = new System.Windows.Forms.CheckBox(); + cbIncludeInternal = new System.Windows.Forms.CheckBox(); + cbImportProjectSpecific = new System.Windows.Forms.CheckBox(); + cbImportDeprecated = new System.Windows.Forms.CheckBox(); + aiImportAll = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + aiInternal = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + aiProjectSpecific = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + aiDeprecated = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(57, 76); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(104, 15); + label1.TabIndex = 0; + label1.Text = "Object Schema ID:"; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(119, 37); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(42, 15); + label2.TabIndex = 1; + label2.Text = "Name:"; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(26, 116); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(135, 15); + label3.TabIndex = 2; + label3.Text = "Data Access Credentials:"; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(69, 152); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(92, 15); + label4.TabIndex = 3; + label4.Text = "Organisation ID:"; + // + // tbName + // + tbName.Location = new System.Drawing.Point(169, 34); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(370, 23); + tbName.TabIndex = 4; + tbName.TextChanged += ValidateForm; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(169, 73); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(370, 23); + tbUrl.TabIndex = 5; + tbUrl.TextChanged += ValidateForm; + // + // tbOrganisationId + // + tbOrganisationId.Location = new System.Drawing.Point(169, 149); + tbOrganisationId.Name = "tbOrganisationId"; + tbOrganisationId.Size = new System.Drawing.Size(370, 23); + tbOrganisationId.TabIndex = 6; + tbOrganisationId.TextChanged += ValidateForm; + // + // cbCredentials + // + cbCredentials.FormattingEnabled = true; + cbCredentials.Location = new System.Drawing.Point(169, 108); + cbCredentials.Name = "cbCredentials"; + cbCredentials.Size = new System.Drawing.Size(208, 23); + cbCredentials.TabIndex = 7; + cbCredentials.SelectedIndexChanged += ValidateForm; + // + // btnSave + // + btnSave.Enabled = false; + btnSave.Location = new System.Drawing.Point(464, 318); + btnSave.Name = "btnSave"; + btnSave.Size = new System.Drawing.Size(75, 23); + btnSave.TabIndex = 8; + btnSave.Text = "Save"; + btnSave.UseVisualStyleBackColor = true; + btnSave.Click += Save; + // + // cbImportCatalogues + // + cbImportCatalogues.AutoSize = true; + cbImportCatalogues.Location = new System.Drawing.Point(337, 188); + cbImportCatalogues.Name = "cbImportCatalogues"; + cbImportCatalogues.Size = new System.Drawing.Size(202, 19); + cbImportCatalogues.TabIndex = 9; + cbImportCatalogues.Text = "Import All Extractable Catalogues"; + cbImportCatalogues.UseVisualStyleBackColor = true; + // + // cbIncludeInternal + // + cbIncludeInternal.AutoSize = true; + cbIncludeInternal.Location = new System.Drawing.Point(337, 213); + cbIncludeInternal.Name = "cbIncludeInternal"; + cbIncludeInternal.Size = new System.Drawing.Size(184, 19); + cbIncludeInternal.TabIndex = 10; + cbIncludeInternal.Text = "Import All Internal Catalogues"; + cbIncludeInternal.UseVisualStyleBackColor = true; + // + // cbImportProjectSpecific + // + cbImportProjectSpecific.AutoSize = true; + cbImportProjectSpecific.Location = new System.Drawing.Point(337, 238); + cbImportProjectSpecific.Name = "cbImportProjectSpecific"; + cbImportProjectSpecific.Size = new System.Drawing.Size(225, 19); + cbImportProjectSpecific.TabIndex = 11; + cbImportProjectSpecific.Text = "Import All Project Sepcific Catalogues"; + cbImportProjectSpecific.UseVisualStyleBackColor = true; + // + // cbImportDeprecated + // + cbImportDeprecated.AutoSize = true; + cbImportDeprecated.Location = new System.Drawing.Point(337, 263); + cbImportDeprecated.Name = "cbImportDeprecated"; + cbImportDeprecated.Size = new System.Drawing.Size(204, 19); + cbImportDeprecated.TabIndex = 12; + cbImportDeprecated.Text = "Import All Deprecated Catalogues"; + cbImportDeprecated.UseVisualStyleBackColor = true; + // + // aiImportAll + // + aiImportAll.Location = new System.Drawing.Point(542, 188); + aiImportAll.Name = "aiImportAll"; + aiImportAll.Size = new System.Drawing.Size(20, 20); + aiImportAll.TabIndex = 13; + // + // aiInternal + // + aiInternal.Location = new System.Drawing.Point(521, 213); + aiInternal.Name = "aiInternal"; + aiInternal.Size = new System.Drawing.Size(20, 20); + aiInternal.TabIndex = 14; + // + // aiProjectSpecific + // + aiProjectSpecific.Location = new System.Drawing.Point(559, 237); + aiProjectSpecific.Name = "aiProjectSpecific"; + aiProjectSpecific.Size = new System.Drawing.Size(20, 20); + aiProjectSpecific.TabIndex = 15; + // + // aiDeprecated + // + aiDeprecated.Location = new System.Drawing.Point(542, 263); + aiDeprecated.Name = "aiDeprecated"; + aiDeprecated.Size = new System.Drawing.Size(20, 20); + aiDeprecated.TabIndex = 16; + // + // CreateNewJiraConfigurationUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(800, 450); + Controls.Add(aiDeprecated); + Controls.Add(aiProjectSpecific); + Controls.Add(aiInternal); + Controls.Add(aiImportAll); + Controls.Add(cbImportDeprecated); + Controls.Add(cbImportProjectSpecific); + Controls.Add(cbIncludeInternal); + Controls.Add(cbImportCatalogues); + Controls.Add(btnSave); + Controls.Add(cbCredentials); + Controls.Add(tbOrganisationId); + Controls.Add(tbUrl); + Controls.Add(tbName); + Controls.Add(label4); + Controls.Add(label3); + Controls.Add(label2); + Controls.Add(label1); + Name = "CreateNewJiraConfigurationUI"; + Text = "Create Jira Configuration"; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.TextBox tbName; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.TextBox tbOrganisationId; + private System.Windows.Forms.ComboBox cbCredentials; + private System.Windows.Forms.Button btnSave; + private System.Windows.Forms.CheckBox cbImportCatalogues; + private System.Windows.Forms.CheckBox cbIncludeInternal; + private System.Windows.Forms.CheckBox cbImportProjectSpecific; + private System.Windows.Forms.CheckBox cbImportDeprecated; + private SimpleControls.AdditionalInfomationUI aiImportAll; + private SimpleControls.AdditionalInfomationUI aiInternal; + private SimpleControls.AdditionalInfomationUI aiProjectSpecific; + private SimpleControls.AdditionalInfomationUI aiDeprecated; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs new file mode 100644 index 0000000000..170e037800 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs @@ -0,0 +1,81 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + public partial class CreateNewDatasetProviderConfigurationUI : RDMPForm + { + private readonly IActivateItems _activator; + private readonly Type _providerType; + public CreateNewDatasetProviderConfigurationUI(IActivateItems activator, Type providerType) + { + InitializeComponent(); + _activator = activator; + _providerType = providerType; + var dataAccessCredentials = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects(); + cbCredentials.Items.Clear(); + cbCredentials.Items.AddRange(dataAccessCredentials); + } + + private void DisableSave() + { + btnSave.Enabled = false; + } + + private void ValidateForm(object sender, EventArgs e) + { + + if (cbCredentials.SelectedItem is null) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbName.Text)) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbUrl.Text)) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbOrganisationId.Text)) + { + DisableSave(); + return; + } + btnSave.Enabled = true; + } + + private void Save(object sender, EventArgs e) + { + var config = new DatasetProviderConfiguration(_activator.RepositoryLocator.CatalogueRepository, tbName.Text, _providerType.FullName, tbUrl.Text, ((DataAccessCredentials)cbCredentials.SelectedItem).ID, tbOrganisationId.Text); + config.SaveToDatabase(); + _activator.Publish(config); + + if (cbImportCatalogues.Checked || cbImportProjectSpecific.Checked || cbIncludeInternal.Checked || cbImportDeprecated.Checked) + { + var provider = config.GetProviderInstance(_activator); + var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(_activator, provider, cbImportCatalogues.Checked, cbIncludeInternal.Checked, cbImportProjectSpecific.Checked, cbImportDeprecated.Checked); + cmd.Execute(); + } + + Close(); + _activator.Show($"Dataset Provider '{tbName.Text}' has successfully been created"); + } + } + +} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs new file mode 100644 index 0000000000..eff75b5aed --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs @@ -0,0 +1,131 @@ +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + partial class ImportExistingDatasetUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + btnCreate = new System.Windows.Forms.Button(); + btnCancel = new System.Windows.Forms.Button(); + tbUrl = new System.Windows.Forms.TextBox(); + lblError = new System.Windows.Forms.Label(); + label2 = new System.Windows.Forms.Label(); + cbProviders = new System.Windows.Forms.ComboBox(); + label1 = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // btnCreate + // + btnCreate.Location = new System.Drawing.Point(550, 108); + btnCreate.Name = "btnCreate"; + btnCreate.Size = new System.Drawing.Size(75, 23); + btnCreate.TabIndex = 14; + btnCreate.Text = "Create"; + btnCreate.UseVisualStyleBackColor = true; + btnCreate.Click += btnCreate_Click; + // + // btnCancel + // + btnCancel.Location = new System.Drawing.Point(459, 108); + btnCancel.Name = "btnCancel"; + btnCancel.Size = new System.Drawing.Size(75, 23); + btnCancel.TabIndex = 15; + btnCancel.Text = "Cancel"; + btnCancel.UseVisualStyleBackColor = true; + btnCancel.Click += btnCancel_Click; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(74, 56); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(551, 23); + tbUrl.TabIndex = 17; + // + // lblError + // + lblError.AutoSize = true; + lblError.ForeColor = System.Drawing.Color.Red; + lblError.Location = new System.Drawing.Point(23, 95); + lblError.Name = "lblError"; + lblError.Size = new System.Drawing.Size(0, 15); + lblError.TabIndex = 18; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(14, 20); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(54, 15); + label2.TabIndex = 19; + label2.Text = "Provider:"; + // + // cbProviders + // + cbProviders.FormattingEnabled = true; + cbProviders.Location = new System.Drawing.Point(74, 17); + cbProviders.Name = "cbProviders"; + cbProviders.Size = new System.Drawing.Size(345, 23); + cbProviders.TabIndex = 20; + cbProviders.SelectedIndexChanged += cbProviders_SelectedIndexChanged; + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(37, 59); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(37, 15); + label1.TabIndex = 16; + label1.Text = "UUID:"; + // + // ImportExistingPureDatasetUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(667, 158); + Controls.Add(cbProviders); + Controls.Add(label2); + Controls.Add(lblError); + Controls.Add(tbUrl); + Controls.Add(label1); + Controls.Add(btnCancel); + Controls.Add(btnCreate); + Name = "ImportExistingPureDatasetUI"; + Text = "Import Existing HDR Dataset"; + Load += ImportExistingPureDatasetUI_Load; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + private System.Windows.Forms.Button btnCreate; + private System.Windows.Forms.Button btnCancel; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.Label lblError; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.ComboBox cbProviders; + private System.Windows.Forms.Label label1; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs new file mode 100644 index 0000000000..2ee5815126 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs @@ -0,0 +1,77 @@ +using Microsoft.Data.SqlClient; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.UI.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + public partial class ImportExistingDatasetUI: RDMPForm + { + private readonly IActivateItems _activator; + private readonly Type _providerType; + + private readonly string[] _visibilities = { "FREE", "CAMPUS", "BACKEND", "CONFIDENTIAL" }; + + private DatasetProviderConfiguration _providerConfiguration; + //private JiraDatasetProvider _datasetProvider; + + public ImportExistingDatasetUI(IActivateItems activator, Type providerType) : base(activator) + { + _activator = activator; + _providerType = providerType; + InitializeComponent(); + var configs = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Type", _providerType.ToString()); + cbProviders.Items.AddRange(configs); + } + + private void ImportExistingPureDatasetUI_Load(object sender, EventArgs e) + { + + } + + private void btnCancel_Click(object sender, EventArgs e) + { + Dispose(); + } + + private void btnCreate_Click(object sender, EventArgs e) + { + if (_activator.YesNo("Please confirm you wish to import this Dataset", "Import Dataset")) + { + try + { + var provider = (cbProviders.SelectedItem as DatasetProviderConfiguration).GetProviderInstance(_activator); + provider.AddExistingDataset(null, tbUrl.Text); + Close(); + Dispose(); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + lblError.Text = "Unable to locate dataset. Ensure UUID is correct."; + lblError.Visible = true; + return; + } + } + } + + private void cbProviders_SelectedIndexChanged(object sender, EventArgs e) + { + if (cbProviders.SelectedItem is DatasetProviderConfiguration config) + { + _providerConfiguration = config; + } + } + } +} + From c975ab8bfaa50a6febc07fe5d196fd90dc760953 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 29 Apr 2025 13:43:59 +0100 Subject: [PATCH 023/142] fix return types --- Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs | 2 +- Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs | 2 +- Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs index 7ad62ce065..1f62633406 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs @@ -24,5 +24,5 @@ public interface IDatasetProvider Dataset Create(Catalogue catalogue); Dataset AddExistingDatasetWithReturn(string name, string url); - Dataset AddExistingDataset(string name, string url); + void AddExistingDataset(string name, string url); } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs index a61f66153d..064a08509b 100644 --- a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs @@ -16,7 +16,7 @@ public InternalDatasetProvider(IBasicActivateItems activator, DatasetProviderCon _activator = activator; } - public Dataset AddExistingDataset(string name, string url) + public void AddExistingDataset(string name, string url) { throw new System.NotImplementedException(); } diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 5e6c3f95be..76dedc203f 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -38,7 +38,7 @@ public virtual Dataset Create(Catalogue catalogue) throw new NotImplementedException(); } - public virtual Dataset AddExistingDataset(string name, string url) + public virtual void AddExistingDataset(string name, string url) { throw new NotImplementedException(); } From 1ec74e1e75931e8f7eb6fe833c77daf36cb31f5f Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 29 Apr 2025 15:08:58 +0100 Subject: [PATCH 024/142] fix autoupdate --- Rdmp.Core/Curation/Data/Catalogue.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 78ee80a344..5a75754ff2 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -1165,7 +1165,7 @@ public override void SaveToDatabase() var repositoryProvider = new UserSettingsRepositoryFinder(); var activator = new ThrowImmediatelyActivator(repositoryProvider, ThrowImmediatelyCheckNotifier.Quiet); var providerInstance = providerConfiguration.GetProviderInstance(activator); - var ds = providerInstance.FetchDatasetByID(int.Parse(dataset.Url.Split('/').Last())); + var ds = providerInstance.FetchDatasetByID(int.Parse(dataset.GetRemoteID())); providerInstance.UpdateUsingCatalogue(ds, this); } } From f51be9122a0e54182ddf2c9f15bc2d0ad18fec78 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 29 Apr 2025 15:28:51 +0100 Subject: [PATCH 025/142] fix update --- Rdmp.Core/Curation/Data/Catalogue.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 5a75754ff2..a05bae570b 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -1165,8 +1165,7 @@ public override void SaveToDatabase() var repositoryProvider = new UserSettingsRepositoryFinder(); var activator = new ThrowImmediatelyActivator(repositoryProvider, ThrowImmediatelyCheckNotifier.Quiet); var providerInstance = providerConfiguration.GetProviderInstance(activator); - var ds = providerInstance.FetchDatasetByID(int.Parse(dataset.GetRemoteID())); - providerInstance.UpdateUsingCatalogue(ds, this); + providerInstance.UpdateUsingCatalogue(dataset, this); } } From 7cb508b70dcf36a02a255e4db6991e19532782a9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Apr 2025 13:30:19 +0100 Subject: [PATCH 026/142] remove unused function --- Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs | 7 ------- .../Curation/Data/Datasets/InternalDatasetProvider.cs | 6 ------ Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs | 2 -- 3 files changed, 15 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs index 1f62633406..fe901bf440 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs @@ -7,13 +7,6 @@ namespace Rdmp.Core.Curation.Data.Datasets; public interface IDatasetProvider { - /// - /// Fetch known datasets - /// - /// - List FetchDatasets(); - - /// /// Fetch a specific dataset by its ID /// diff --git a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs index 064a08509b..ddd7156eee 100644 --- a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs @@ -37,12 +37,6 @@ public Curation.Data.Datasets.Dataset FetchDatasetByID(int id) return _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", id).FirstOrDefault(); } - /// - public List FetchDatasets() - { - return _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); - } - public void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) { throw new System.NotImplementedException(); diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 76dedc203f..3bab0882e2 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -24,8 +24,6 @@ protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderCo public abstract Dataset FetchDatasetByID(int id); - public abstract List FetchDatasets(); - public abstract Dataset AddExistingDatasetWithReturn(string name, string url); From 5c6e6c189948a6f5e50939c1861d73b070385edd Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Apr 2025 13:38:20 +0100 Subject: [PATCH 027/142] fix split regex --- Rdmp.UI/Collections/ConfigurationsCollectionUI.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index 20dfe69886..9151193be9 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -36,16 +36,16 @@ private IAtomicCommand[] GetWhitespaceRightClickMenu() OverrideCommandName="Add New Regex Redaction Configuration" } }; - foreach(var provider in datasetProviders) + foreach (var provider in datasetProviders) { - options = options.Append(new ExecuteCommandAddNewDatasetProviderUI(_activator,provider) + options = options.Append(new ExecuteCommandAddNewDatasetProviderUI(_activator, provider) { - OverrideCommandName = $"Add New {System.Text.RegularExpressions.Regex.Replace(provider.Name, "([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim()}", + OverrideCommandName = $"Add New {System.Text.RegularExpressions.Regex.Replace(provider.Name, "(?<=[a-z])([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim()}", SuggestedCategory = "Dataset Provider Configurations" }).ToArray(); options = options.Append(new ExecuteCommandAddNewDatasetUI(_activator, provider) { - OverrideCommandName = $"Add Existing {System.Text.RegularExpressions.Regex.Replace(provider.Name, "([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim().Replace("Provider","")}", + OverrideCommandName = $"Add Existing {System.Text.RegularExpressions.Regex.Replace(provider.Name, "(?<=[a-z])([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim().Replace("Provider", "")}", SuggestedCategory = "Datasets" }).ToArray(); } @@ -65,7 +65,7 @@ public override void SetItemActivator(IActivateItems activator) tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetProviderConfigurationsNode); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllRegexRedactionConfigurationsNode); tlvConfigurations.Refresh(); - } + } public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) { From aa47b8b2062cdd0b7ac797878d826190155cb02a Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 1 May 2025 09:55:05 +0100 Subject: [PATCH 028/142] fix up --- Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs index 9f4f59d492..bd07409e71 100644 --- a/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs +++ b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs @@ -35,10 +35,10 @@ private void button1_Click(object sender, EventArgs e) { var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(_activator); var dataset = provider.Create(_catalogue); - - if (!_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Dataset_ID", dataset.ID).Where(l => l.Catalogue.ID == _catalogue.ID).Any()) + var ds = provider.AddExistingDatasetWithReturn(null, dataset.GetID()); + if (!_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Dataset_ID", ds.ID).Where(l => l.Catalogue.ID == _catalogue.ID).Any()) { - var linkage = new CatalogueDatasetLinkage(_activator.RepositoryLocator.CatalogueRepository, _catalogue, dataset); + var linkage = new CatalogueDatasetLinkage(_activator.RepositoryLocator.CatalogueRepository, _catalogue, ds); linkage.SaveToDatabase(); } Close(); From fdaee38a4e301f5ec812f24d1bc16b42a00ae76c Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 6 May 2025 08:08:48 +0100 Subject: [PATCH 029/142] tidy up --- ...ngCataloguesIntoExternalDatasetProvider.cs | 7 ++-- .../ExecuteCommandLinkCatalogueToDataset.cs | 8 ++-- .../Collections/ConfigurationsCollectionUI.cs | 38 ++++++++++++++++++- Rdmp.UI/MainFormUITabs/CatalogueUI.cs | 10 ++--- ...DatasetProviderConfigurationUI.Designer.cs | 4 +- ...CreateNewDatasetProviderConfigurationUI.cs | 3 +- .../ImportExistingDatasetUI.Designer.cs | 2 +- .../LinkDatasetDialog.Designer.cs | 2 +- Rdmp.UI/SubComponents/LinkDatasetDialog.cs | 2 + 9 files changed, 55 insertions(+), 21 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs index 5a52e9255b..5e5f1fc8d5 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -18,8 +18,9 @@ public class ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider : private readonly bool _includeInternal; private readonly bool _includeProjectSpecific; private readonly bool _includeDeprecated; + private readonly bool _autoUpdate; - public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicActivateItems activator, IDatasetProvider provider, bool includeExtractable, bool includeInternal, bool includeProjectSpecific, bool includeDeprecated) + public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicActivateItems activator, IDatasetProvider provider, bool includeExtractable, bool includeInternal, bool includeProjectSpecific, bool includeDeprecated, bool autoUpdate) { _activator = activator; _provider = provider; @@ -27,6 +28,7 @@ public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicA _includeInternal = includeInternal; _includeProjectSpecific = includeProjectSpecific; _includeDeprecated = includeDeprecated; + _autoUpdate = autoUpdate; } @@ -49,12 +51,11 @@ public override void Execute() { catalogues = catalogues.Where(c => !c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList(); } - //todo check this catalogue filtering works foreach (var catalogue in catalogues) { var dataset = _provider.Create(catalogue); var ds = _provider.AddExistingDatasetWithReturn(null, dataset.GetID()); - var cmd = new ExecuteCommandLinkCatalogueToDataset(_activator, catalogue, ds); + var cmd = new ExecuteCommandLinkCatalogueToDataset(_activator, catalogue, ds,_autoUpdate); cmd.Execute(); } } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs index d5b571c872..69d8d3b945 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs @@ -15,12 +15,12 @@ public sealed class ExecuteCommandLinkCatalogueToDataset : BasicCommandExecution { private readonly Catalogue _catalogue; private readonly Curation.Data.Datasets.Dataset _dataset; - private readonly bool _linkAll; - public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [DemandsInitialization("The catalogue To link")]Catalogue catalogue, [DemandsInitialization("The dataset to link to")]Curation.Data.Datasets.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) + private readonly bool _autoUpdate; + public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [DemandsInitialization("The catalogue To link")]Catalogue catalogue, [DemandsInitialization("The dataset to link to")]Curation.Data.Datasets.Dataset dataset, bool autoUpdate = true) : base(activator) { _catalogue = catalogue; _dataset = dataset; - _linkAll = linkAllOtherColumns; + _autoUpdate = autoUpdate; if (_catalogue is null) SetImpossible("No Catalogue Selected"); if (_dataset is null) SetImpossible("No Dataset Selected"); @@ -30,7 +30,7 @@ public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [Dema public override void Execute() { base.Execute(); - var linkage = new CatalogueDatasetLinkage(_catalogue.CatalogueRepository, _catalogue, _dataset); + var linkage = new CatalogueDatasetLinkage(_catalogue.CatalogueRepository, _catalogue, _dataset,_autoUpdate); linkage.SaveToDatabase(); Publish(linkage); diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index 9151193be9..e82d408bfd 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -9,6 +9,7 @@ using Rdmp.Core.Curation.Data.Datasets; using System; using static Rdmp.Core.Curation.Data.Catalogue; +using System.Text; namespace Rdmp.UI.Collections; @@ -22,6 +23,39 @@ public ConfigurationsCollectionUI() InitializeComponent(); } + private static string FormatPascalAndAcronym(string input) + { + var builder = new StringBuilder(input[0].ToString()); + if (builder.Length > 0) + { + for (var index = 1; index < input.Length; index++) + { + char prevChar = input[index - 1]; + char nextChar = index + 1 < input.Length ? input[index + 1] : '\0'; + + bool isNextLower = Char.IsLower(nextChar); + bool isNextUpper = Char.IsUpper(nextChar); + bool isPresentUpper = Char.IsUpper(input[index]); + bool isPrevLower = Char.IsLower(prevChar); + bool isPrevUpper = Char.IsUpper(prevChar); + + if (!string.IsNullOrWhiteSpace(prevChar.ToString()) && + ((isPrevUpper && isPresentUpper && isNextLower) || + (isPrevLower && isPresentUpper && isNextLower) || + (isPrevLower && isPresentUpper && isNextUpper))) + { + builder.Append(' '); + builder.Append(input[index]); + } + else + { + builder.Append(input[index]); + } + } + } + return builder.ToString(); + } + private IAtomicCommand[] GetWhitespaceRightClickMenu() { var datasetProviders = AppDomain.CurrentDomain.GetAssemblies().SelectMany(s => s.GetTypes()).Where(p => typeof(PluginDatasetProvider).IsAssignableFrom(p) && !p.IsAbstract); @@ -40,12 +74,12 @@ private IAtomicCommand[] GetWhitespaceRightClickMenu() { options = options.Append(new ExecuteCommandAddNewDatasetProviderUI(_activator, provider) { - OverrideCommandName = $"Add New {System.Text.RegularExpressions.Regex.Replace(provider.Name, "(?<=[a-z])([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim()}", + OverrideCommandName = $"Add New {FormatPascalAndAcronym(provider.Name).Trim()}", SuggestedCategory = "Dataset Provider Configurations" }).ToArray(); options = options.Append(new ExecuteCommandAddNewDatasetUI(_activator, provider) { - OverrideCommandName = $"Add Existing {System.Text.RegularExpressions.Regex.Replace(provider.Name, "(?<=[a-z])([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim().Replace("Provider", "")}", + OverrideCommandName = $"Add Existing {FormatPascalAndAcronym(provider.Name).Trim().Replace("Provider", "")}", SuggestedCategory = "Datasets" }).ToArray(); } diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs index 04ff867703..eeda3fb5cd 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs @@ -456,6 +456,7 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) aiAssociatedMedia.SetItemActivator(Activator); break; case 7: + tableLayoutPanel3.SuspendLayout(); var datasets = _catalogue.GetLinkedDatasets(); for (int i = 1; i < tableLayoutPanel3.RowCount; i++) { @@ -482,13 +483,6 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) btn.Text = "Update Now"; btn.Click += (object sender, EventArgs e) => UpdateDataset(dataset); tableLayoutPanel3.Controls.Add(btn, 2, tableLayoutPanel3.RowCount - 1); - //if (dataset.Type != typeof(JiraDatasetProvider).ToString()) - //{ - // var btn2 = new Button(); - // btn2.Text = "View"; - // btn2.Click += (object sender, EventArgs e) => UsefulStuff.OpenUrl(dataset.Url); - // tableLayoutPanel3.Controls.Add(btn2, 3, tableLayoutPanel3.RowCount - 1); - //} var btn3 = new Button(); btn3.Text = "Remove Link to Dataset"; btn3.Click += (object sender, EventArgs e) => @@ -505,6 +499,7 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) tableLayoutPanel3.Controls.Add(labelType, 5, tableLayoutPanel3.RowCount - 1); } + tableLayoutPanel3.ResumeLayout(); break; default: break; @@ -517,6 +512,7 @@ private void UpdateDataset(Dataset dataset) var providerConfiguration = Activator.RepositoryLocator.CatalogueRepository.GetObjectByID((int)dataset.Provider_ID); var provider = providerConfiguration.GetProviderInstance(Activator); provider.UpdateUsingCatalogue(dataset, _catalogue); + Publish(_catalogue); } private void tableLayoutPanel3_Paint(object sender, PaintEventArgs e) { diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs index d87e84e69f..2597bd6636 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs @@ -54,7 +54,7 @@ private void InitializeComponent() label1.Name = "label1"; label1.Size = new System.Drawing.Size(104, 15); label1.TabIndex = 0; - label1.Text = "Object Schema ID:"; + label1.Text = "URL/ID:"; // // label2 // @@ -218,7 +218,7 @@ private void InitializeComponent() Controls.Add(label2); Controls.Add(label1); Name = "CreateNewJiraConfigurationUI"; - Text = "Create Jira Configuration"; + Text = "Create New Dataset Provider Configuration"; ResumeLayout(false); PerformLayout(); } diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs index 170e037800..c870f95970 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs @@ -69,7 +69,8 @@ private void Save(object sender, EventArgs e) if (cbImportCatalogues.Checked || cbImportProjectSpecific.Checked || cbIncludeInternal.Checked || cbImportDeprecated.Checked) { var provider = config.GetProviderInstance(_activator); - var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(_activator, provider, cbImportCatalogues.Checked, cbIncludeInternal.Checked, cbImportProjectSpecific.Checked, cbImportDeprecated.Checked); + var autoUpdate = _activator.YesNo("Do you want these dataset to be updated automatically?", "Automatically Update Datasets?"); + var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(_activator, provider, cbImportCatalogues.Checked, cbIncludeInternal.Checked, cbImportProjectSpecific.Checked, cbImportDeprecated.Checked, autoUpdate); cmd.Execute(); } diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs index eff75b5aed..f6eeac2276 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs @@ -113,7 +113,7 @@ private void InitializeComponent() Controls.Add(btnCancel); Controls.Add(btnCreate); Name = "ImportExistingPureDatasetUI"; - Text = "Import Existing HDR Dataset"; + Text = "Import Existing Dataset"; Load += ImportExistingPureDatasetUI_Load; ResumeLayout(false); PerformLayout(); diff --git a/Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs b/Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs index 9a811a97f7..14cbf2ae48 100644 --- a/Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs +++ b/Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs @@ -91,7 +91,7 @@ private void InitializeComponent() Controls.Add(groupBox3); Controls.Add(groupBox2); Name = "LinkDatasetDialog"; - Text = "Link Dataset"; + Text = "Link Existing Dataset"; groupBox2.ResumeLayout(false); groupBox2.PerformLayout(); groupBox3.ResumeLayout(false); diff --git a/Rdmp.UI/SubComponents/LinkDatasetDialog.cs b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs index e7a98c5036..a21b17e411 100644 --- a/Rdmp.UI/SubComponents/LinkDatasetDialog.cs +++ b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs @@ -10,6 +10,7 @@ using System.Text; using System.Threading.Tasks; using System.Windows.Forms; +using System.Security.Policy; namespace Rdmp.UI.SubComponents { @@ -47,6 +48,7 @@ private void button1_Click(object sender, EventArgs e) var linkage = new CatalogueDatasetLinkage(_activator.RepositoryLocator.CatalogueRepository, _catalogue, dataset); linkage.SaveToDatabase(); } + _activator.Publish(_catalogue); Close(); } From 4ea67a4acc120256d1b9839c948a49b0a7f76775 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 6 May 2025 08:44:30 +0100 Subject: [PATCH 030/142] fix up ui --- Rdmp.UI/MainFormUITabs/CatalogueUI.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs index eeda3fb5cd..2921c0bbf9 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs @@ -156,6 +156,8 @@ private void RefreshUIFromDatabase() ticketingControl1.Enabled = true; ticketingControl1.TicketText = _catalogue.Ticket; + setTabBindings.Clear(); + tabControl1_SelectedIndexChanged(tabControl1, null); } @@ -458,9 +460,9 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) case 7: tableLayoutPanel3.SuspendLayout(); var datasets = _catalogue.GetLinkedDatasets(); - for (int i = 1; i < tableLayoutPanel3.RowCount; i++) + while (tableLayoutPanel3.Controls.Count > 0) { - tableLayoutPanel3.RowStyles.RemoveAt(i); + tableLayoutPanel3.Controls[0].Dispose(); } foreach (var dataset in datasets) { @@ -490,6 +492,8 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) if (Activator.YesNo("Are you sure?", "Remove Linkage")) { linkage.DeleteInDatabase(); + setTabBindings.Remove(7); + tabControl1_SelectedIndexChanged(tabControl1, null); } }; tableLayoutPanel3.Controls.Add(btn3, 4, tableLayoutPanel3.RowCount - 1); From b247ca37f1feea059d96f1cdb3f98159d5bdd1cf Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 6 May 2025 09:04:19 +0100 Subject: [PATCH 031/142] tidy up and add tests --- Rdmp.Core/CommandExecution/AtomicCommandFactory.cs | 2 ++ ...tExistingCataloguesIntoExternalDatasetProvider.cs | 10 ++++++++-- Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs | 12 +++--------- Rdmp.UI/Collections/ConfigurationsCollectionUI.cs | 3 --- .../Datasets/ImportExistingDatasetUI.cs | 10 ---------- .../SubComponents/DatasetProviderConfigurationUI.cs | 5 ----- Rdmp.UI/SubComponents/LinkDatasetDialog.cs | 2 +- 7 files changed, 14 insertions(+), 30 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 38d67ded93..c118d6fad5 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -19,6 +19,7 @@ using Rdmp.Core.Curation.Data.Cache; using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Curation.Data.Defaults; using Rdmp.Core.Curation.Data.Pipelines; using Rdmp.Core.Curation.Data.Referencing; @@ -41,6 +42,7 @@ using Rdmp.Core.ReusableLibraryCode.DataAccess; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; +using static Microsoft.IO.RecyclableMemoryStreamManager; namespace Rdmp.Core.CommandExecution; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs index 5e5f1fc8d5..a3a572722d 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -31,8 +31,7 @@ public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicA _autoUpdate = autoUpdate; } - - public override void Execute() + public List GetCatalogues() { var catalogues = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); if (!_includeInternal) @@ -51,6 +50,13 @@ public override void Execute() { catalogues = catalogues.Where(c => !c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList(); } + return catalogues; + } + + + public override void Execute() + { + var catalogues = GetCatalogues(); foreach (var catalogue in catalogues) { var dataset = _provider.Create(catalogue); diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs index 1949d5c355..526e99c91f 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs @@ -7,20 +7,14 @@ namespace Rdmp.Core.Curation.Data.Datasets /// /// Base class to allow all plugin dataset types to be based off /// - public class PluginDataset : Rdmp.Core.Curation.Data.Datasets.Dataset + public abstract class PluginDataset : Rdmp.Core.Curation.Data.Datasets.Dataset { public PluginDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name) { } public PluginDataset() { } public PluginDataset(ICatalogueRepository repository, DbDataReader r) : base(repository, r) { } - public override string GetID() - { - return ID.ToString(); - } - public override string GetRemoteID() - { - return ID.ToString(); - } + public override abstract string GetID(); + public override abstract string GetRemoteID(); } } \ No newline at end of file diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index e82d408bfd..0f6940d973 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -62,9 +62,6 @@ private IAtomicCommand[] GetWhitespaceRightClickMenu() var options = new IAtomicCommand[] { - new ExecuteCommandCreateNewDatasetUI(_activator){ - OverrideCommandName="Add New Dataset" - }, new ExecuteCommandAddNewRegexRedactionConfigurationUI(_activator) { OverrideCommandName="Add New Regex Redaction Configuration" diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs index 2ee5815126..6e84677ea2 100644 --- a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs @@ -19,12 +19,6 @@ public partial class ImportExistingDatasetUI: RDMPForm { private readonly IActivateItems _activator; private readonly Type _providerType; - - private readonly string[] _visibilities = { "FREE", "CAMPUS", "BACKEND", "CONFIDENTIAL" }; - - private DatasetProviderConfiguration _providerConfiguration; - //private JiraDatasetProvider _datasetProvider; - public ImportExistingDatasetUI(IActivateItems activator, Type providerType) : base(activator) { _activator = activator; @@ -67,10 +61,6 @@ private void btnCreate_Click(object sender, EventArgs e) private void cbProviders_SelectedIndexChanged(object sender, EventArgs e) { - if (cbProviders.SelectedItem is DatasetProviderConfiguration config) - { - _providerConfiguration = config; - } } } } diff --git a/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs index f5aacbf2c1..c2c24a99dd 100644 --- a/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs @@ -32,11 +32,6 @@ public override void SetDatabaseObject(IActivateItems activator, DatasetProvider tbName.Text = _configuration.Name; tbOrgId.Text = _configuration.Organisation_ID; tbType.Text = _configuration.Type; - //if (_configuration.Type == typeof(JiraDatasetProvider).ToString()) - //{ - // label3.Text = "Object Schema:"; - // label3.Location = new Point(label3.Location.X - 50, label3.Location.Y); - //} tbUrl.Text = _configuration.Url; cbAccessCredentials.Items.Clear(); var credentials = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToArray(); diff --git a/Rdmp.UI/SubComponents/LinkDatasetDialog.cs b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs index a21b17e411..bb33cf05dc 100644 --- a/Rdmp.UI/SubComponents/LinkDatasetDialog.cs +++ b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs @@ -34,7 +34,7 @@ private void button1_Click(object sender, EventArgs e) { var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(_activator); Dataset dataset; - var fetchedDataset = provider.FetchDatasetByID(int.Parse(tbID.Text));//todo is it always ints? + var fetchedDataset = provider.FetchDatasetByID(int.Parse(tbID.Text)); if (_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.Url).Any()) { dataset = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.Url).First(); From c4ac978ed99cddc009619cd23c51f687bf43f4ec Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 6 May 2025 09:26:33 +0100 Subject: [PATCH 032/142] fix abstract --- Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs index 526e99c91f..6959d67570 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs @@ -1,4 +1,5 @@ -using Rdmp.Core.Repositories; +using NPOI.OpenXmlFormats.Dml; +using Rdmp.Core.Repositories; using System.Data.Common; @@ -7,14 +8,19 @@ namespace Rdmp.Core.Curation.Data.Datasets /// /// Base class to allow all plugin dataset types to be based off /// - public abstract class PluginDataset : Rdmp.Core.Curation.Data.Datasets.Dataset + public class PluginDataset : Rdmp.Core.Curation.Data.Datasets.Dataset { public PluginDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name) { } public PluginDataset() { } public PluginDataset(ICatalogueRepository repository, DbDataReader r) : base(repository, r) { } - public override abstract string GetID(); - public override abstract string GetRemoteID(); + public override string GetID() { + return ID.ToString(); + } + public override string GetRemoteID() + { + return ID.ToString(); + } } } \ No newline at end of file From e99cc19a3eb9519b03da4e48fff2716d280b62c7 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 6 May 2025 10:36:51 +0100 Subject: [PATCH 033/142] add view --- .../Data/Datasets/IDatasetProvider.cs | 2 + .../Data/Datasets/InternalDatasetProvider.cs | 5 + .../Curation/Data/Datasets/PluginDataset.cs | 1 + .../Data/Datasets/PluginDatasetProvider.cs | 5 + .../ProposeExecutionWhenTargetIsDataset.cs | 7 +- .../DatasetConfigurationUI.Designer.cs | 124 ------------------ .../SubComponents/DatasetConfigurationUI.cs | 70 ---------- .../SubComponents/DatasetConfigurationUI.resx | 120 ----------------- 8 files changed, 19 insertions(+), 315 deletions(-) delete mode 100644 Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs delete mode 100644 Rdmp.UI/SubComponents/DatasetConfigurationUI.cs delete mode 100644 Rdmp.UI/SubComponents/DatasetConfigurationUI.resx diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs index fe901bf440..017d216784 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs @@ -18,4 +18,6 @@ public interface IDatasetProvider Dataset AddExistingDatasetWithReturn(string name, string url); void AddExistingDataset(string name, string url); + + string GetRemoteURL(Dataset dataset); } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs index ddd7156eee..420c7e8e0d 100644 --- a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs @@ -37,6 +37,11 @@ public Curation.Data.Datasets.Dataset FetchDatasetByID(int id) return _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", id).FirstOrDefault(); } + public string GetRemoteURL(Dataset dataset) + { + throw new System.NotImplementedException(); + } + public void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) { throw new System.NotImplementedException(); diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs index 6959d67570..887b68e403 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs @@ -1,5 +1,6 @@ using NPOI.OpenXmlFormats.Dml; using Rdmp.Core.Repositories; +using System.CodeDom; using System.Data.Common; diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 3bab0882e2..338af94ee4 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -40,4 +40,9 @@ public virtual void AddExistingDataset(string name, string url) { throw new NotImplementedException(); } + + public string GetRemoteURL(Dataset dataset) + { + throw new NotImplementedException(); + } } \ No newline at end of file diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs index 22f816d991..c4d0a03376 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsDataset.cs @@ -6,6 +6,7 @@ using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.ReusableLibraryCode; using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.UI.ItemActivation; using Rdmp.UI.SubComponents; @@ -25,7 +26,11 @@ public ProposeExecutionWhenTargetIsDataset(IActivateItems itemActivator) : base( public override void Activate(Dataset target) { - ItemActivator.Activate(target); + var providerConfiguration = ItemActivator.RepositoryLocator.CatalogueRepository.GetObjectByID((int)target.Provider_ID); + var provider = providerConfiguration.GetProviderInstance(ItemActivator); + var remoteURL = provider.GetRemoteURL(target); + if(remoteURL != null) + UsefulStuff.OpenUrl(remoteURL); } [CanBeNull] diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs deleted file mode 100644 index b251b593bc..0000000000 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace Rdmp.UI.SubComponents; - -partial class DatasetConfigurationUI -{ - /// - /// Required designer variable. - /// - private System.ComponentModel.IContainer components = null; - - /// - /// Clean up any resources being used. - /// - /// true if managed resources should be disposed; otherwise, false. - protected override void Dispose(bool disposing) - { - if (disposing && (components != null)) - { - components.Dispose(); - } - base.Dispose(disposing); - } - - #region Windows Form Designer generated code - - /// - /// Required method for Designer support - do not modify - /// the contents of this method with the code editor. - /// - private void InitializeComponent() - { - label1 = new System.Windows.Forms.Label(); - tbName = new System.Windows.Forms.TextBox(); - tbDOI = new System.Windows.Forms.TextBox(); - tbSource = new System.Windows.Forms.TextBox(); - label2 = new System.Windows.Forms.Label(); - label3 = new System.Windows.Forms.Label(); - lblDatasetUsage = new System.Windows.Forms.Label(); - SuspendLayout(); - // - // label1 - // - label1.AutoSize = true; - label1.Location = new System.Drawing.Point(24, 40); - label1.Name = "label1"; - label1.Size = new System.Drawing.Size(86, 15); - label1.TabIndex = 0; - label1.Text = "Dataset Name*"; - label1.Click += label1_Click; - // - // tbName - // - tbName.Location = new System.Drawing.Point(27, 65); - tbName.Name = "tbName"; - tbName.Size = new System.Drawing.Size(270, 23); - tbName.TabIndex = 1; - // - // tbDOI - // - tbDOI.Location = new System.Drawing.Point(24, 155); - tbDOI.Name = "tbDOI"; - tbDOI.Size = new System.Drawing.Size(273, 23); - tbDOI.TabIndex = 2; - // - // tbSource - // - tbSource.Location = new System.Drawing.Point(24, 235); - tbSource.Name = "tbSource"; - tbSource.Size = new System.Drawing.Size(273, 23); - tbSource.TabIndex = 3; - // - // label2 - // - label2.AutoSize = true; - label2.Location = new System.Drawing.Point(24, 123); - label2.Name = "label2"; - label2.Size = new System.Drawing.Size(69, 15); - label2.TabIndex = 4; - label2.Text = "Dataset DOI"; - // - // label3 - // - label3.AutoSize = true; - label3.Location = new System.Drawing.Point(28, 202); - label3.Name = "label3"; - label3.Size = new System.Drawing.Size(85, 15); - label3.TabIndex = 5; - label3.Text = "Dataset Source"; - // - // lblDatasetUsage - // - lblDatasetUsage.AutoSize = true; - lblDatasetUsage.Location = new System.Drawing.Point(369, 65); - lblDatasetUsage.Name = "lblDatasetUsage"; - lblDatasetUsage.Size = new System.Drawing.Size(38, 15); - lblDatasetUsage.TabIndex = 8; - lblDatasetUsage.Text = "label5"; - // - // DatasetConfigurationUI - // - AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); - AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - Controls.Add(lblDatasetUsage); - Controls.Add(label3); - Controls.Add(label2); - Controls.Add(tbSource); - Controls.Add(tbDOI); - Controls.Add(tbName); - Controls.Add(label1); - Name = "DatasetConfigurationUI"; - Size = new System.Drawing.Size(800, 450); - ResumeLayout(false); - PerformLayout(); - } - - #endregion - - private System.Windows.Forms.Label label1; - private System.Windows.Forms.TextBox tbName; - private System.Windows.Forms.TextBox tbDOI; - private System.Windows.Forms.TextBox tbSource; - private System.Windows.Forms.Label label2; - private System.Windows.Forms.Label label3; - private System.Windows.Forms.Label lblDatasetUsage; -} diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs b/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs deleted file mode 100644 index aa44e9aab7..0000000000 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs +++ /dev/null @@ -1,70 +0,0 @@ -using Rdmp.UI.TestsAndSetup.ServicePropogation; -using System; -using System.ComponentModel; -using System.Linq; -using System.Windows.Forms; -using Rdmp.UI.Refreshing; -using Rdmp.Core.Dataset; -using Rdmp.Core.Curation.Data; -using Rdmp.UI.ItemActivation; -using Rdmp.Core.Curation.Data.Datasets; - -namespace Rdmp.UI.SubComponents; -public partial class DatasetConfigurationUI : DatsetConfigurationUI_Design, IRefreshBusSubscriber -{ - private readonly DatasetConfigurationUICommon _common; - - public DatasetConfigurationUI() - { - InitializeComponent(); - _common = new DatasetConfigurationUICommon(); - } - - public override void SetDatabaseObject(IActivateItems activator, Dataset databaseObject) - { - base.SetDatabaseObject(activator, databaseObject); - _common.Dataset = databaseObject; - - var catalogues = databaseObject.CatalogueRepository - .GetAllObjectsWhere("Dataset_ID", databaseObject.ID).SelectMany(static ci => ci.CatalogueItems) - .Select(static ci => ci.CatalogueName).Distinct().ToList(); - if(catalogues.Count < 1) - { - lblDatasetUsage.Text = "This dataset is not linked to data yet."; - } - else - { - var catalogueString = string.Join(Environment.NewLine, catalogues); - lblDatasetUsage.Text = $"This dataset is used in the following catalogues:{Environment.NewLine}{catalogueString}"; - } - - Bind(tbName, "Text", "Name", static c => c.Name); - Bind(tbDOI, "Text", "DigitalObjectIdentifier", static c => c.DigitalObjectIdentifier); - Bind(tbSource, "Text", "Source", static c => c.Source); - var s = GetObjectSaverButton(); - s.SetupFor(this, databaseObject, activator); - GetObjectSaverButton()?.Enable(false); - - } - - - public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) - { - } - - private void label1_Click(object sender, EventArgs e) - { - - } - - private void label4_Click(object sender, EventArgs e) - { - - } -} -[TypeDescriptionProvider( - typeof(AbstractControlDescriptionProvider))] -public abstract class - DatsetConfigurationUI_Design : RDMPSingleDatabaseObjectControl -{ -} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.resx b/Rdmp.UI/SubComponents/DatasetConfigurationUI.resx deleted file mode 100644 index af32865ec1..0000000000 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.resx +++ /dev/null @@ -1,120 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - \ No newline at end of file From fee8ade2a675a1da8c8f596562cce771fbade94f Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 6 May 2025 10:55:50 +0100 Subject: [PATCH 034/142] add virtual --- Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 338af94ee4..32eb9970bf 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -41,7 +41,7 @@ public virtual void AddExistingDataset(string name, string url) throw new NotImplementedException(); } - public string GetRemoteURL(Dataset dataset) + public virtual string GetRemoteURL(Dataset dataset) { throw new NotImplementedException(); } From c94b11592161c54375e2f2af8937ed31d6ee4f31 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 12 May 2025 11:04:29 +0100 Subject: [PATCH 035/142] add tests --- ...aloguesIntoExternalDatasetProviderTests.cs | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs new file mode 100644 index 0000000000..414a1009aa --- /dev/null +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs @@ -0,0 +1,62 @@ +using NUnit.Framework; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.DataExport.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Tests.CommandExecution +{ + class ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests: CommandCliTests + { + [Test] + public void TestImportInternalCataloguesOnly() + { + var _cata1 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset1"); + _cata1.IsInternalDataset = true; + var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2"); + _cata1.SaveToDatabase(); + _cata2.SaveToDatabase(); + + var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test",1,"test"); + var provider = new InternalDatasetProvider(GetMockActivator(),configuration,null); + var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false,true, false, false,false); + Assert.That(cmd.GetCatalogues().Count, Is.EqualTo(1)); + } + [Test] + public void TestImportProjectSpecificCataloguesOnly() + { + var _cata1 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset1"); + + var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2"); + _cata1.SaveToDatabase(); + _cata2.SaveToDatabase(); + + var ext = new ExtractableDataSet(GetMockActivator().RepositoryLocator.DataExportRepository, _cata1); + ext.SaveToDatabase(); + + var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test", 1, "test"); + var provider = new InternalDatasetProvider(GetMockActivator(), configuration, null); + var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false, false, true, false, false); + Assert.That(cmd.GetCatalogues().Count, Is.EqualTo(1)); + } + [Test] + public void TestImportDeprecatedCataloguesOnly() + { + var _cata1 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset1"); + _cata1.IsDeprecated = true; + var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2"); + _cata1.SaveToDatabase(); + _cata2.SaveToDatabase(); + + var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test", 1, "test"); + var provider = new InternalDatasetProvider(GetMockActivator(), configuration, null); + var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false, false, false, true, false); + Assert.That(cmd.GetCatalogues().Count, Is.EqualTo(1)); + } + } +} From e8423f085292b5315dcf3d42a702aae2c471c618 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 25 Jun 2025 11:05:34 +0100 Subject: [PATCH 036/142] add basic dataset variable document --- .../Pipeline/ExtractionPipelineUseCase.cs | 13 ++++ .../DatasetVariableReportGenerator.cs | 71 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/ExtractionPipelineUseCase.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/ExtractionPipelineUseCase.cs index 87f95b82a9..faa03834e6 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/ExtractionPipelineUseCase.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/ExtractionPipelineUseCase.cs @@ -316,6 +316,19 @@ private void WriteMetadata(IDataLoadEventListener listener) new NotifyEventArgs(ProgressEventType.Error, "Word metadata document NOT CREATED", e)); return; } + try + { + var datasetVariableReportWriter = new DatasetVariableReportGenerator(this); + datasetVariableReportWriter.GenerateDatasetVariableReport(); + } + catch (Exception e) + { + ExtractCommand.ElevateState(ExtractCommandState.Warning); + + listener.OnNotify(this, + new NotifyEventArgs(ProgressEventType.Error, "Dataset variable document NOT CREATED", e)); + return; + } //if there were any exceptions if (wordDataWriter.ExceptionsGeneratingWordFile.Any()) diff --git a/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs new file mode 100644 index 0000000000..682190d88d --- /dev/null +++ b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs @@ -0,0 +1,71 @@ +using CommandLine; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.DataExport.DataExtraction.Pipeline; +using Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; +using Rdmp.Core.QueryBuilding; +using System; +using System.CodeDom.Compiler; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Reports.ExtractionTime +{ + class DatasetVariableReportGenerator + { + private ICatalogue _catalogue; + private ExtractionPipelineUseCase _executor; + private IExecuteDatasetExtractionDestination _destination; + private List _columnstoExtract; + private List _releaseSubs; + + + public DatasetVariableReportGenerator(ExtractionPipelineUseCase executer) { + _executor = executer; + _destination = executer.Destination; + _catalogue = executer.Source.Request.Catalogue; + _columnstoExtract = executer.Source.Request.ColumnsToExtract.Select(c => c.Cast()).ToList(); + _releaseSubs = executer.Source.Request.ReleaseIdentifierSubstitutions; + } + + public void GenerateDatasetVariableReport() + { + var csv = new StringBuilder(); + WriteHeaders(csv); + foreach(var column in _columnstoExtract) + { + WriteColumn(column, csv); + } + foreach (var sub in _releaseSubs) + { + WriteReleaseSubs(sub, csv); + } + File.WriteAllText(new FileInfo(Path.Combine(_destination.DirectoryPopulated.FullName, + $"{_destination.GetFilename()}Variables.csv")).ToString(), csv.ToString()); + } + + private void WriteColumn(ExtractableColumn column, StringBuilder sb) + { + var catalogueItem = _catalogue.CatalogueItems.Where(c => c.ColumnInfo_ID == column.ColumnInfo.ID).First(); + bool isNull = !catalogueItem.ExtractionInformation.IsPrimaryKey; + bool isIdentifier = catalogueItem.ExtractionInformation.IsExtractionIdentifier; + sb.AppendLine($"\"{column.GetRuntimeName()}\",\"{column.ColumnInfo.Data_type}\",{isNull},\"{catalogueItem.Description}\",{isIdentifier}"); + } + + private void WriteReleaseSubs(ReleaseIdentifierSubstitution releaseIdentifierSubstitution,StringBuilder sb) + { + var column = releaseIdentifierSubstitution.ColumnInfo; + var catalogueItem = _catalogue.CatalogueItems.Where(c => c.ColumnInfo_ID == column.ID).First(); + sb.AppendLine($"\"{releaseIdentifierSubstitution.Alias}\",\"{column.Data_type}\",{false},\"{catalogueItem.Description}\",{releaseIdentifierSubstitution.IsExtractionIdentifier}"); + + } + + private void WriteHeaders(StringBuilder sb) + { + sb.AppendLine("Variable Name, Type, Null possible(Y/N),Description,Identifier"); + } + } +} From bffe9f8dbcd44289ae0ad5dbb98dbcac916ec83c Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 25 Jun 2025 14:34:20 +0100 Subject: [PATCH 037/142] improve lookups --- CHANGELOG.md | 1 + .../DatasetVariableReportGenerator.cs | 24 ++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c67d991bd4..122cd7c742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix issue with global contextual search - Add MS Teams Extraction notifications (see [Documentation\DataExtractions\ExtractionTeamsNotifications.md]) - Add ability to use Cohort Catalogue Filters in an Extraction Configuration +- Add Dataset Variable document to extractions ## [8.4.4] - 2025-05-08 diff --git a/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs index 682190d88d..89e84eb55f 100644 --- a/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs +++ b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs @@ -1,4 +1,5 @@ using CommandLine; +using Rdmp.Core.CohortCommitting; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction.Pipeline; @@ -47,25 +48,42 @@ public void GenerateDatasetVariableReport() $"{_destination.GetFilename()}Variables.csv")).ToString(), csv.ToString()); } + + private string LookupKeyNameGenerator(ColumnInfo columnInfo) + { + var split = columnInfo.GetFullyQualifiedName().Split('.'); + return $"{split[^2]}.{split[^1]}"; + } + private string LookupStringGenerator(Lookup lookup) + { + return $"{LookupKeyNameGenerator(lookup.ForeignKey)} = {LookupKeyNameGenerator(lookup.PrimaryKey)}"; + } + private void WriteColumn(ExtractableColumn column, StringBuilder sb) { var catalogueItem = _catalogue.CatalogueItems.Where(c => c.ColumnInfo_ID == column.ColumnInfo.ID).First(); bool isNull = !catalogueItem.ExtractionInformation.IsPrimaryKey; bool isIdentifier = catalogueItem.ExtractionInformation.IsExtractionIdentifier; - sb.AppendLine($"\"{column.GetRuntimeName()}\",\"{column.ColumnInfo.Data_type}\",{isNull},\"{catalogueItem.Description}\",{isIdentifier}"); + var lookups = _catalogue.CatalogueRepository.GetAllObjectsWhere("ForeignKey_ID",column.ColumnInfo.ID); + var lookupString = ""; + if (lookups.Any()) lookupString = string.Join(';', lookups.Select(l => LookupStringGenerator(l))); + sb.AppendLine($"\"{column.GetRuntimeName()}\",\"{column.ColumnInfo.Data_type}\",{isNull},\"{catalogueItem.Description}\",{isIdentifier},{lookups.Any()},{lookupString}"); } private void WriteReleaseSubs(ReleaseIdentifierSubstitution releaseIdentifierSubstitution,StringBuilder sb) { var column = releaseIdentifierSubstitution.ColumnInfo; var catalogueItem = _catalogue.CatalogueItems.Where(c => c.ColumnInfo_ID == column.ID).First(); - sb.AppendLine($"\"{releaseIdentifierSubstitution.Alias}\",\"{column.Data_type}\",{false},\"{catalogueItem.Description}\",{releaseIdentifierSubstitution.IsExtractionIdentifier}"); + var lookups = _catalogue.CatalogueRepository.GetAllObjectsWhere("ForeignKey_ID", column.ID); + var lookupString = ""; + if (lookups.Any()) lookupString = string.Join(';', lookups.Select(l => LookupStringGenerator(l))); + sb.AppendLine($"\"{releaseIdentifierSubstitution.Alias}\",\"{column.Data_type}\",{false},\"{catalogueItem.Description}\",{releaseIdentifierSubstitution.IsExtractionIdentifier},{lookups.Any()},{lookupString}"); } private void WriteHeaders(StringBuilder sb) { - sb.AppendLine("Variable Name, Type, Null possible(Y/N),Description,Identifier"); + sb.AppendLine("Variable Name, Type, Null possible(Y/N),Description,Identifier,HasLookups,Lookups"); } } } From 4692bd6364cdc811ec052dc88b9f7feccc4f69e9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 24 Jul 2025 14:35:16 +0100 Subject: [PATCH 038/142] speed up catalogues --- .../Curation/ProjectspecificCatalogueTests.cs | 5 +- .../CommandExecution/AtomicCommandFactory.cs | 38 +++++++-------- .../ExecuteCommandAddNewAggregateGraph.cs | 3 +- .../ExecuteCommandAddNewCatalogueItem.cs | 8 +++- .../ExecuteCommandChangeExtractability.cs | 6 ++- ...cuteCommandMakeCatalogueProjectSpecific.cs | 32 ++++++++----- ...MakeProjectSpecificCatalogueNormalAgain.cs | 6 +-- .../ExecuteCommandSetColumnSettingBase.cs | 11 +++-- .../ExecuteCommandExportInDublinCoreFormat.cs | 8 ++-- .../CommandExecution/GoToCommandFactory.cs | 48 +++++++++---------- Rdmp.Core/Curation/Data/Catalogue.cs | 1 + .../Data/ProjectSpecificCatalogueManager.cs | 10 ++-- Rdmp.Core/Providers/CatalogueChildProvider.cs | 6 ++- Rdmp.Core/Providers/ICoreChildProvider.cs | 1 + .../ExecuteCommandShowTooltip.cs | 2 +- ...cuteCommandViewCatalogueExtractionSqlUI.cs | 6 ++- .../ExecuteCommandViewCommits.cs | 10 ++-- Rdmp.UI/Menus/CatalogueMenu.cs | 8 ++-- Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs | 10 ++-- Rdmp.UI/Menus/RDMPContextMenuStrip.cs | 6 +-- 20 files changed, 127 insertions(+), 98 deletions(-) diff --git a/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs b/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs index e6630495c7..cc679bcb66 100644 --- a/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs +++ b/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs @@ -5,6 +5,7 @@ using Rdmp.Core.CommandLine.Interactive; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; +using Rdmp.Core.Providers; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Checks; using System; @@ -92,14 +93,14 @@ private void MakeProjectSpecific(Catalogue catalogue, Project project, List var cmd = new ExecuteCommandMakeCatalogueProjectSpecific(_activator,catalogue,project,force); cmd.SetTarget(project); cmd.SetTarget(catalogue); - if (shouldThrow) Assert.That(ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(_activator.RepositoryLocator.DataExportRepository,catalogue,project,projectIdsToIgnore??new List()), Is.False); + if (shouldThrow) Assert.That(ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific((DataExportChildProvider)_activator.CoreChildProvider,catalogue,project,projectIdsToIgnore??new List()), Is.False); else Assert.DoesNotThrow(() => cmd.Execute()); } private void MakeNotProjectSpecific(Catalogue catalogue, ExtractableDataSet eds, Project project, bool shouldThrow = false) { var cmd = new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, catalogue, eds); - if (shouldThrow) Assert.That(ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(_activator.RepositoryLocator.DataExportRepository, catalogue, eds,project), Is.False); + if (shouldThrow) Assert.That(ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(catalogue, eds,project), Is.False); else Assert.DoesNotThrow(() => cmd.Execute()); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 7d01c6ea01..798459b1bb 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -135,16 +135,16 @@ public IEnumerable CreateCommands(object o) SuggestedCategory = Extraction }; - yield return new ExecuteCommandMakeCatalogueProjectSpecific(_activator, c, null,false) + yield return new ExecuteCommandMakeCatalogueProjectSpecific(_activator, c, null, false) { Weight = -99.0009f, SuggestedCategory = Extraction }; - yield return new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, c,null) + yield return new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, c, null) { Weight = -99.0009f, SuggestedCategory = Extraction, - OverrideCommandName="Remove Project Specific Catalogue from a Project" + OverrideCommandName = "Remove Project Specific Catalogue from a Project" }; yield return new ExecuteCommandSetExtractionIdentifier(_activator, c, null, null) @@ -899,22 +899,22 @@ public IEnumerable CreateCommands(object o) }; } - if (Is(o, out IMightBeDeprecated d)) - { - yield return new ExecuteCommandDeprecate(_activator, new[] { d }, !d.IsDeprecated) - { - OverrideCommandName = d.IsDeprecated ? "Un Deprecate" : "Deprecate", - SuggestedCategory = Deprecation, - Weight = -99.7f - }; - yield return new ExecuteCommandReplacedBy(_activator, d, null) - { - PromptToPickReplacement = true, - SuggestedCategory = Deprecation, - Weight = -99.6f, - OverrideCommandName = "Set Replaced By" - }; - } + //if (Is(o, out IMightBeDeprecated d)) + //{ + // yield return new ExecuteCommandDeprecate(_activator, new[] { d }, !d.IsDeprecated) + // { + // OverrideCommandName = d.IsDeprecated ? "Un Deprecate" : "Deprecate", + // SuggestedCategory = Deprecation, + // Weight = -99.7f + // }; + // yield return new ExecuteCommandReplacedBy(_activator, d, null) + // { + // PromptToPickReplacement = true, + // SuggestedCategory = Deprecation, + // Weight = -99.6f, + // OverrideCommandName = "Set Replaced By" + // }; + //} if (Is(o, out CohortAggregateContainer cohortAggregateContainer)) { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs index 29bb64882d..6d996b414b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs @@ -25,8 +25,7 @@ public ExecuteCommandAddNewAggregateGraph(IBasicActivateItems activator, Catalog { _catalogue = catalogue; _name = name; - if (_catalogue != null && _catalogue.GetAllExtractionInformation(ExtractionCategory.Any) - .All(ei => ei.ColumnInfo == null)) + if (_catalogue != null && activator.CoreChildProvider.AllExtractionInformations.Where(ei => ei.CatalogueItem.Catalogue_ID == catalogue.ID && ei.ExtractionCategory == ExtractionCategory.Any).All(ei => ei.ColumnInfo == null)) SetImpossible("Catalogue has no extractable columns"); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs index 7d9580e68f..518da26bfe 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.Linq; +using NPOI.OpenXmlFormats; using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.Curation.Data; using Rdmp.Core.Icons.IconProvision; @@ -23,6 +24,7 @@ public class ExecuteCommandAddNewCatalogueItem : BasicCommandExecution, IAtomicC private readonly Catalogue _catalogue; private readonly ColumnInfo[] _columnInfos; private readonly HashSet _existingColumnInfos; + private readonly IBasicActivateItems _activator; /// /// The category to assign for newly created . @@ -33,12 +35,14 @@ public class ExecuteCommandAddNewCatalogueItem : BasicCommandExecution, IAtomicC public ExecuteCommandAddNewCatalogueItem(IBasicActivateItems activator, Catalogue catalogue, ColumnInfoCombineable colInfo) : this(activator, catalogue, colInfo.ColumnInfos) { + _activator = activator; } [UseWithObjectConstructor] public ExecuteCommandAddNewCatalogueItem(IBasicActivateItems activator, Catalogue catalogue, params ColumnInfo[] columnInfos) : base(activator) { + _activator = activator; _catalogue = catalogue; _existingColumnInfos = GetColumnInfos(_catalogue); _columnInfos = columnInfos; @@ -48,11 +52,11 @@ public ExecuteCommandAddNewCatalogueItem(IBasicActivateItems activator, Catalogu SetImpossible("ColumnInfo(s) are already in Catalogue"); } - private static HashSet GetColumnInfos(ICatalogue catalogue) + private HashSet GetColumnInfos(ICatalogue catalogue) { return catalogue == null ? null - : new HashSet(catalogue.CatalogueItems.Select(static ci => ci.ColumnInfo_ID) + : new HashSet(_activator.CoreChildProvider.AllCatalogueItems.Where(ci => ci.Catalogue_ID == catalogue.ID).Select(static ci => ci.ColumnInfo_ID) .Where(static col => col.HasValue).Select(static v => v.Value).Distinct()); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs index f27befd4a3..7a05adb0da 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs @@ -23,8 +23,12 @@ public sealed class ExecuteCommandChangeExtractability : BasicCommandExecution public ExecuteCommandChangeExtractability(IBasicActivateItems activator, Catalogue catalogue, bool? explicitExtractability = null) : base(activator) { + //calls db but not sure how to improve this _catalogue = catalogue; - var status = catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository); + var dataExportChildProvider = (DataExportChildProvider)activator.CoreChildProvider; + var eds = dataExportChildProvider.ExtractableDataSets.Where(eds => eds.Catalogue_ID == catalogue.ID).ToList(); + var status = eds.Count == 0 ? new CatalogueExtractabilityStatus(false, false) : new CatalogueExtractabilityStatus(true, eds.Count > 1 ? true : eds.First().Projects.Any()); + if (status == null) { SetImpossible( diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs index 8039dbaeae..0d5622f0cf 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs @@ -4,16 +4,17 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Linq; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.Providers; using Rdmp.Core.Repositories.Construction; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; +using System; +using System.Collections.Generic; +using System.Linq; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -23,6 +24,7 @@ public class ExecuteCommandMakeCatalogueProjectSpecific : BasicCommandExecution, private IProject _project; private List _existingProjectIDs; private readonly bool _force = false; + private readonly IBasicActivateItems _activator; [UseWithObjectConstructor] public ExecuteCommandMakeCatalogueProjectSpecific(IBasicActivateItems itemActivator, ICatalogue catalogue, @@ -31,11 +33,13 @@ public ExecuteCommandMakeCatalogueProjectSpecific(IBasicActivateItems itemActiva SetCatalogue(catalogue); _project = project; _force = force; + _activator = itemActivator; } public ExecuteCommandMakeCatalogueProjectSpecific(IBasicActivateItems itemActivator) : base(itemActivator) { UseTripleDotSuffix = true; + _activator = itemActivator; } public override string GetCommandHelp() => @@ -45,7 +49,8 @@ public override void Execute() { if (_catalogue == null) SetCatalogue(SelectOne(BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList())); - GetExistingProjectIDs(); + if(_existingProjectIDs is null) + GetExistingProjectIDs(); _project ??= SelectOne(GetListOfValidProjects()); @@ -79,20 +84,23 @@ public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) private List GetListOfValidProjects() { - var availableProjects = BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(p => !p.GetAllProjectCatalogues().Contains(_catalogue)); - return availableProjects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(BasicActivator.RepositoryLocator.DataExportRepository, _catalogue, p, _existingProjectIDs)).ToList(); + var dataExportChildProvider = ((DataExportChildProvider)_activator.CoreChildProvider); + + var availableProjects = dataExportChildProvider.Projects.Where(p =>!dataExportChildProvider.ExtractableDataSetProjects.Where(edsp => edsp.Project_ID == p.ID).Select(edsp => edsp.DataSet.Catalogue).Contains(_catalogue)); + return availableProjects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(dataExportChildProvider, _catalogue, p, _existingProjectIDs)).ToList(); } private void GetExistingProjectIDs() - { - var existingProjects = BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(p => p.GetAllProjectCatalogues().Contains(_catalogue)); + { + var dataExportChildProvider = ((DataExportChildProvider)_activator.CoreChildProvider); + var existingProjects = dataExportChildProvider.Projects.Where(p => dataExportChildProvider.ExtractableDataSetProjects.Where(edsp => edsp.Project_ID==p.ID).Select(edsp => edsp.DataSet.Catalogue).Contains(_catalogue)); _existingProjectIDs = existingProjects.Select(p => p.ID).ToList(); } private void SetCatalogue(ICatalogue catalogue) { ResetImpossibleness(); - + var dataExportChildProvider = ((DataExportChildProvider)_activator.CoreChildProvider); _catalogue = catalogue; GetExistingProjectIDs(); if (catalogue == null) @@ -100,8 +108,8 @@ private void SetCatalogue(ICatalogue catalogue) SetImpossible("Catalogue cannot be null"); return; } - - var status = _catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository); + var eds = dataExportChildProvider.ExtractableDataSets.Where(eds => eds.Catalogue_ID == catalogue.ID).ToList(); + var status = eds.Count == 0 ? new CatalogueExtractabilityStatus(false, false) : new CatalogueExtractabilityStatus(true, eds.Count > 1 ? true : eds.First().Projects.Any()); if (!GetListOfValidProjects().Any() && !_force) { @@ -111,7 +119,7 @@ private void SetCatalogue(ICatalogue catalogue) if (!status.IsExtractable) SetImpossible("Catalogue must first be made Extractable"); - var ei = _catalogue.GetAllExtractionInformation(ExtractionCategory.Any); + var ei = dataExportChildProvider.AllCatalogueItems.Where(ci => ci.Catalogue_ID == _catalogue.ID && ci.ExtractionInformation != null).Select(ci => ci.ExtractionInformation); if (!ei.Any()) SetImpossible("Catalogue has no extractable columns"); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs index a8a0511c3f..348649a090 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs @@ -36,8 +36,8 @@ public ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(IBasicActivateItems SetImpossible("Data Export functionality is not available"); return; } - - _extractableDataSets = dataExportRepository.GetAllObjectsWithParent(catalogue).Where(eds => eds.Projects.Any() && eds.Projects.Select(p =>ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(dataExportRepository, catalogue, eds,p)).Contains(true)).ToList(); + + _extractableDataSets = dataExportRepository.GetAllObjectsWithParent(catalogue).Where(eds => eds.Projects.Any() && eds.Projects.Select(p =>ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(catalogue, eds,p)).Contains(true)).ToList(); if (!_extractableDataSets.Any()) { SetImpossible("Cannot make Catalogue Non-Project specific"); @@ -57,7 +57,7 @@ public override void Execute() { var dataExportRepository = BasicActivator.RepositoryLocator.DataExportRepository; - var projectIds = _extractableDataSets.SelectMany(eds => eds.Projects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(dataExportRepository,_catalogue,eds,p))).Select(p => p.ID); + var projectIds = _extractableDataSets.SelectMany(eds => eds.Projects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(_catalogue,eds,p))).Select(p => p.ID); _selectedProj = SelectOne(BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjectsInIDList(projectIds).ToList()); if (_selectedProj is null) return; _extractableDataSet = _extractableDataSets.FirstOrDefault(eds => eds.Projects.Select(p => p.ID).Contains(_selectedProj.ID)); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs index 5599c403c9..3fae936b7d 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs @@ -4,10 +4,11 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Linq; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; +using Rdmp.Core.Providers; +using System; +using System.Linq; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -48,12 +49,14 @@ public ExecuteCommandSetColumnSettingBase( _commandName = commandName; _catalogue.ClearAllInjections(); _commandProperty = commandProperty; + var dataExportChildProvider = ((DataExportChildProvider)activator.CoreChildProvider); + if (inConfiguration != null) { SetImpossibleIfReadonly(_inConfiguration); - var allEds = inConfiguration.GetAllExtractableDataSets(); + var allEds = dataExportChildProvider.ExtractableDataSets.Where(eds => eds.ExtractionConfigurations.Contains(inConfiguration)); var eds = allEds.FirstOrDefault(sds => sds.Catalogue_ID == _catalogue.ID); if (eds == null) { @@ -73,7 +76,7 @@ public ExecuteCommandSetColumnSettingBase( } else { - _extractionInformations = _catalogue.GetAllExtractionInformation(ExtractionCategory.Any); + _extractionInformations = dataExportChildProvider.AllCatalogueItems.Where(ci => ci.Catalogue_ID == catalogue.ID && ci.ExtractionInformation != null && ci.ExtractionInformation.ExtractionCategory == ExtractionCategory.Any).Select(ci => ci.ExtractionInformation).ToArray(); if (_extractionInformations.Length == 0) { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/Sharing/ExecuteCommandExportInDublinCoreFormat.cs b/Rdmp.Core/CommandExecution/AtomicCommands/Sharing/ExecuteCommandExportInDublinCoreFormat.cs index ab5bbcde5c..1a7bbaca08 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/Sharing/ExecuteCommandExportInDublinCoreFormat.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/Sharing/ExecuteCommandExportInDublinCoreFormat.cs @@ -12,19 +12,19 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands.Sharing; public class ExecuteCommandExportInDublinCoreFormat : BasicCommandExecution, IAtomicCommand { - private readonly DublinCoreDefinition _definition; + private readonly Catalogue _catalogue; private FileInfo _toExport; - private readonly DublinCoreTranslater _translater = new(); public ExecuteCommandExportInDublinCoreFormat(IBasicActivateItems activator, Catalogue catalogue) : base(activator) { - _definition = DublinCoreTranslater.GenerateFrom(catalogue); - UseTripleDotSuffix = true; + _catalogue = catalogue; + UseTripleDotSuffix = true; } public override void Execute() { base.Execute(); + DublinCoreDefinition _definition = DublinCoreTranslater.GenerateFrom(_catalogue); if ((_toExport ??= BasicActivator.SelectFile("Dublin Core Xml|*.xml")) == null) return; diff --git a/Rdmp.Core/CommandExecution/GoToCommandFactory.cs b/Rdmp.Core/CommandExecution/GoToCommandFactory.cs index 1d5b8b0d2d..bade8a3eb1 100644 --- a/Rdmp.Core/CommandExecution/GoToCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/GoToCommandFactory.cs @@ -44,29 +44,29 @@ public IEnumerable GetCommands(object forObject) if (Is(forObject, out IInjectKnown ii)) ii.ClearAllInjections(); - if (Is(forObject, out IMapsDirectlyToDatabaseTable mt)) - { - // Go to import / export definitions - var export = _activator.RepositoryLocator.CatalogueRepository.GetReferencesTo(mt) - .FirstOrDefault(); + //if (Is(forObject, out IMapsDirectlyToDatabaseTable mt)) + //{ + // // Go to import / export definitions + // var export = _activator.RepositoryLocator.CatalogueRepository.GetReferencesTo(mt) + // .FirstOrDefault(); - if (export != null) - yield return new ExecuteCommandShow(_activator, export, 0, true) - { OverrideCommandName = "Show Export Definition" }; + // if (export != null) + // yield return new ExecuteCommandShow(_activator, export, 0, true) + // { OverrideCommandName = "Show Export Definition" }; - var import = _activator.RepositoryLocator.CatalogueRepository.GetReferencesTo(mt) - .FirstOrDefault(); - if (import != null) - yield return new ExecuteCommandShow(_activator, import, 0) - { OverrideCommandName = "Show Import Definition" }; + // var import = _activator.RepositoryLocator.CatalogueRepository.GetReferencesTo(mt) + // .FirstOrDefault(); + // if (import != null) + // yield return new ExecuteCommandShow(_activator, import, 0) + // { OverrideCommandName = "Show Import Definition" }; - if (SupportsReplacement(forObject)) - yield return new ExecuteCommandShow(_activator, () => GetReplacementIfAny(mt)) - { OverrideCommandName = "Replacement" }; + // if (SupportsReplacement(forObject)) + // yield return new ExecuteCommandShow(_activator, () => GetReplacementIfAny(mt)) + // { OverrideCommandName = "Replacement" }; - yield return new ExecuteCommandSimilar(_activator, mt, false) { GoTo = true }; - } + // yield return new ExecuteCommandSimilar(_activator, mt, false) { GoTo = true }; + //} // cic => associated projects if (Is(forObject, out CohortIdentificationConfiguration cic)) @@ -188,14 +188,12 @@ public IEnumerable GetCommands(object forObject) if (Is(forObject, out ExtractionFilter masterFilter)) { yield return new ExecuteCommandShow(_activator, () => - _activator.RepositoryLocator.CatalogueRepository - .GetAllObjectsWhere("ClonedFromExtractionFilter_ID", masterFilter.ID) - .Select(f => f.GetAggregate()) + _activator.CoreChildProvider.AllAggregateFilters.Where(af => af.ClonedFromExtractionFilter_ID == masterFilter.ID).Select(f => f.GetAggregate()) .Where(a => a != null).Distinct() ) { OverrideCommandName = "Usages (in Cohort Builder)" }; - + //TODO yield return new ExecuteCommandShow(_activator, () => _activator.RepositoryLocator.DataExportRepository .GetAllObjectsWhere("ClonedFromExtractionFilter_ID", masterFilter.ID) @@ -264,12 +262,14 @@ public IEnumerable GetCommands(object forObject) if (Is(forObject, out Catalogue catalogue)) { - foreach (var lmd in catalogue.LoadMetadatas()) + var lmdLinkage = _activator.CoreChildProvider.AllLoadMetadataCatalogueLinkages.Where(lmdcl => lmdcl.CatalogueID == catalogue.ID).Select(lmdcl => lmdcl.LoadMetadataID); + var lmds = _activator.CoreChildProvider.AllLoadMetadatas.Where(lmd => lmdLinkage.Contains(lmd.ID)); + foreach (var lmd in lmds) { yield return new ExecuteCommandShow(_activator, lmd.ID, typeof(LoadMetadata)) { OverrideCommandName = $"Data Load ({lmd.Name})", OverrideIcon = GetImage(RDMPConcept.LoadMetadata) }; } - if (catalogue.LoadMetadatas().Length == 0) + if (!lmds.Any()) { yield return new ExecuteCommandShow(_activator, null, typeof(LoadMetadata)) { OverrideCommandName = "No Data Load", OverrideIcon = GetImage(RDMPConcept.LoadMetadata) }; diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 804c87f6f2..8c3da32b74 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -559,6 +559,7 @@ public DateTime? DatasetStartDate /// public LoadMetadata[] LoadMetadatas() { + var loadMetadataLinkIDs = Repository.GetAllObjectsWhere("CatalogueID", ID).Select(l => l.LoadMetadataID); return Repository.GetAllObjects().Where(cat => loadMetadataLinkIDs.Contains(cat.ID)).ToArray(); diff --git a/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs b/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs index 19884744de..6512c0883c 100644 --- a/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs +++ b/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs @@ -1,6 +1,7 @@ using NPOI.OpenXmlFormats.Spreadsheet; using Org.BouncyCastle.Crypto.Signers; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Providers; using Rdmp.Core.Repositories; using Spectre.Console; using System; @@ -15,9 +16,10 @@ namespace Rdmp.Core.DataExport.Data static class ProjectSpecificCatalogueManager { - public static bool CanMakeCatalogueProjectSpecific(IDataExportRepository dqeRepo, ICatalogue catalogue, IProject project, List projectIdsToIgnore) + public static bool CanMakeCatalogueProjectSpecific(DataExportChildProvider childProvider, ICatalogue catalogue, IProject project, List projectIdsToIgnore) { - var status = catalogue.GetExtractabilityStatus(dqeRepo); + var foundeds = childProvider.ExtractableDataSets.Where(eds => eds.Catalogue_ID == catalogue.ID).ToList(); + var status = foundeds.Count == 0 ? new CatalogueExtractabilityStatus(false, false) : new CatalogueExtractabilityStatus(true, foundeds.Count > 1 ? true : foundeds.First().Projects.Any()); if (!status.IsExtractable) return false; @@ -27,7 +29,7 @@ public static bool CanMakeCatalogueProjectSpecific(IDataExportRepository dqeRepo return false; if (ei.Count(e => e.IsExtractionIdentifier) < 1) return false; - var edss = dqeRepo.GetAllObjectsWithParent(catalogue); + var edss = childProvider.ExtractableDataSets.Where(eds => eds.Catalogue_ID == catalogue.ID).ToList(); if (edss.Any(e => e.Projects.Select(p => p.ID).Contains(project.ID))) { //already project specific @@ -62,7 +64,7 @@ public static ExtractableDataSet MakeCatalogueProjectSpecific(IDataExportReposit return eds; } - public static bool CanMakeCatalogueNonProjectSpecific(IDataExportRepository dqeRepo, ICatalogue catalogue, ExtractableDataSet extractableDataSet,IProject project) + public static bool CanMakeCatalogueNonProjectSpecific(ICatalogue catalogue, ExtractableDataSet extractableDataSet,IProject project) { bool usedInExtraction = project.ExtractionConfigurations.Where(ec => ec.SelectedDataSets.Select(sds => sds.ExtractableDataSet).Contains(extractableDataSet)).Any(); if (!usedInExtraction) return true; diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 16ae2f55ee..7affafdf39 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -55,8 +55,9 @@ public class CatalogueChildProvider : ICoreChildProvider { //Load System public LoadMetadata[] AllLoadMetadatas { get; set; } + public LoadMetadataCatalogueLinkage[] AllLoadMetadataCatalogueLinkages { get; set; } - private LoadMetadataCatalogueLinkage[] AllLoadMetadataLinkage { get; set; } + private LoadMetadataCatalogueLinkage[] AllLoadMetadataLinkage { get; set; } public ProcessTask[] AllProcessTasks { get; set; } public ProcessTaskArgument[] AllProcessTasksArguments { get; set; } @@ -256,7 +257,8 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] AllDatasets = GetAllObjects(repository); AllLoadMetadatas = GetAllObjects(repository); - AllLoadMetadataLinkage = GetAllObjects(repository); + AllLoadMetadataCatalogueLinkages = GetAllObjects(repository); + AllLoadMetadataLinkage = GetAllObjects(repository); AllProcessTasks = GetAllObjects(repository); AllProcessTasksArguments = GetAllObjects(repository); AllLoadProgresses = GetAllObjects(repository); diff --git a/Rdmp.Core/Providers/ICoreChildProvider.cs b/Rdmp.Core/Providers/ICoreChildProvider.cs index 911b02a738..4982e68a1f 100644 --- a/Rdmp.Core/Providers/ICoreChildProvider.cs +++ b/Rdmp.Core/Providers/ICoreChildProvider.cs @@ -35,6 +35,7 @@ public interface ICoreChildProvider : IChildProvider { JoinInfo[] AllJoinInfos { get; } LoadMetadata[] AllLoadMetadatas { get; } + LoadMetadataCatalogueLinkage[] AllLoadMetadataCatalogueLinkages { get; } TableInfoServerNode[] AllServers { get; } TableInfo[] AllTableInfos { get; } Dictionary> TableInfosToColumnInfos { get; } diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandShowTooltip.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandShowTooltip.cs index 90fb7f56dc..63c8e4245e 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandShowTooltip.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandShowTooltip.cs @@ -24,7 +24,7 @@ public class ExecuteCommandShowTooltip : BasicUICommandExecution, IAtomicCommand public ExecuteCommandShowTooltip(IActivateItems activator, object o) : base(activator) { Weight = 100.5f; - + //this REALLY queries the db var hasOne = RDMPCollectionCommonFunctionality.GetToolTip(activator, o, out _title, out _body, out _isBad); if (!hasOne) diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCatalogueExtractionSqlUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCatalogueExtractionSqlUI.cs index cb81d51765..09b3c95367 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCatalogueExtractionSqlUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCatalogueExtractionSqlUI.cs @@ -20,15 +20,18 @@ namespace Rdmp.UI.CommandExecution.AtomicCommands; public class ExecuteCommandViewCatalogueExtractionSqlUI : BasicUICommandExecution, IAtomicCommandWithTarget { private Catalogue _catalogue; + private IActivateItems _actvator; [UseWithObjectConstructor] public ExecuteCommandViewCatalogueExtractionSqlUI(IActivateItems activator, Catalogue catalogue) : this(activator) { _catalogue = catalogue; + _actvator = activator; } public ExecuteCommandViewCatalogueExtractionSqlUI(IActivateItems activator) : base(activator) { + _actvator = activator; } public override string GetCommandHelp() => @@ -40,8 +43,9 @@ public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) { _catalogue = (Catalogue)target; + //if the catalogue has no extractable columns - if (!_catalogue.GetAllExtractionInformation(ExtractionCategory.Any).Any()) + if(_actvator.CoreChildProvider.AllCatalogueItems.Where(ci => ci.Catalogue_ID == _catalogue.ID).Where(ci => ci.ExtractionInformation != null).Any(ci => ci.ExtractionInformation.ExtractionCategory == ExtractionCategory.Any)) SetImpossible("Catalogue has no ExtractionInformations"); return this; diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCommits.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCommits.cs index 13f55f8d52..840fb969dd 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCommits.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCommits.cs @@ -25,11 +25,11 @@ public ExecuteCommandViewCommits(IActivateItems activator, IMapsDirectlyToDataba _o = o; OverrideCommandName = "View History"; - if ( - !activator.RepositoryLocator.CatalogueRepository - .GetAllObjectsWhere(nameof(Memento.ReferencedObjectID), o.ID) - .Any(m => m.IsReferenceTo(o))) - SetImpossible("No commits have been made yet"); + //if ( + // !activator.RepositoryLocator.CatalogueRepository + // .GetAllObjectsWhere(nameof(Memento.ReferencedObjectID), o.ID) + // .Any(m => m.IsReferenceTo(o))) + // SetImpossible("No commits have been made yet"); } public override Image GetImage(IIconProvider iconProvider) => iconProvider.GetImage(RDMPConcept.Commit); diff --git a/Rdmp.UI/Menus/CatalogueMenu.cs b/Rdmp.UI/Menus/CatalogueMenu.cs index 5e511a2965..e9b6a31b64 100644 --- a/Rdmp.UI/Menus/CatalogueMenu.cs +++ b/Rdmp.UI/Menus/CatalogueMenu.cs @@ -77,7 +77,7 @@ public CatalogueMenu(RDMPContextMenuStripArgs args, Catalogue catalogue) : base( Add(new ExecuteCommandReOrderColumns(_activator, catalogue) { SuggestedCategory = CatalogueItems, Weight = -99.046f }); Add(new ExecuteCommandRegexRedaction(_activator, catalogue) - { SuggestedCategory = CatalogueItems, Weight = -99.046f, OverrideCommandName="Regex Redactions" }); + { SuggestedCategory = CatalogueItems, Weight = -99.046f, OverrideCommandName = "Regex Redactions" }); Add(new ExecuteCommandGuessAssociatedColumns(_activator, catalogue, null) { SuggestedCategory = CatalogueItems, Weight = -99.045f, PromptForPartialMatching = true }); Add(new ExecuteCommandChangeExtractionCategory(_activator, @@ -86,9 +86,11 @@ public CatalogueMenu(RDMPContextMenuStripArgs args, Catalogue catalogue) : base( Add(new ExecuteCommandImportCatalogueItemDescriptions(_activator, catalogue, null /*pick at runtime*/) { SuggestedCategory = CatalogueItems, Weight = -99.043f }); - if (!catalogue.LoadMetadatas().Any()) + + var links = _activator.CoreChildProvider.AllLoadMetadataCatalogueLinkages.Where(lmdcl => lmdcl.CatalogueID == catalogue.ID).Select(lmdcl => lmdcl.LoadMetadataID); + if (!links.Any()) { - foreach (var lmd in catalogue.LoadMetadatas()) + foreach (var lmd in _activator.CoreChildProvider.AllLoadMetadatas.Where(lmd => links.Contains(lmd.ID))) { if (lmd.GetRootDirectory() == null) return; try diff --git a/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs b/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs index a84f719b12..9689e8cbb0 100644 --- a/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs +++ b/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs @@ -20,23 +20,21 @@ namespace Rdmp.UI.Menus.MenuItems; internal class DQEMenuItem : RDMPToolStripMenuItem { private readonly Catalogue _catalogue; - private readonly IExternalDatabaseServer _dqeServer; + private IExternalDatabaseServer _dqeServer; public DQEMenuItem(IActivateItems activator, Catalogue catalogue) : base(activator, "Data Quality Engine...") { - _catalogue = catalogue; - - _dqeServer = activator.RepositoryLocator.CatalogueRepository.GetDefaultFor(PermissableDefaults.DQE); + _catalogue = catalogue; Image = activator.CoreIconProvider.GetImage(RDMPConcept.DQE).ImageToBitmap(); - Text = _dqeServer == null ? "Create DQE Database..." : "Data Quality Engine..."; + Text = "Data Quality Engine...";//_dqeServer == null ? "Create DQE Database..." : "Data Quality Engine..."; } protected override void OnClick(EventArgs e) { base.OnClick(e); - + _dqeServer = _activator.RepositoryLocator.CatalogueRepository.GetDefaultFor(PermissableDefaults.DQE); if (_dqeServer == null) { var cmdCreateDb = new ExecuteCommandCreateNewExternalDatabaseServer(_activator, diff --git a/Rdmp.UI/Menus/RDMPContextMenuStrip.cs b/Rdmp.UI/Menus/RDMPContextMenuStrip.cs index 61963b3966..5d58a16dab 100644 --- a/Rdmp.UI/Menus/RDMPContextMenuStrip.cs +++ b/Rdmp.UI/Menus/RDMPContextMenuStrip.cs @@ -165,11 +165,11 @@ public void AddCommonMenuItems(RDMPCollectionCommonFunctionality commonFunctiona if (_o is IMapsDirectlyToDatabaseTable m) Add(new ExecuteCommandViewCommits(_activator, m)); - //ensure all submenus appear in the same place + ////ensure all submenus appear in the same place foreach (var mi in _subMenuDictionary.Values) Items.Add(mi); - //add plugin menu items + ////add plugin menu items foreach (var plugin in _activator.PluginUserInterfaces) try { @@ -207,7 +207,7 @@ public void AddCommonMenuItems(RDMPCollectionCommonFunctionality commonFunctiona if (databaseEntity != null) Add(new ExecuteCommandRefreshObject(_activator, databaseEntity), Keys.F5); - Add(new ExecuteCommandShowTooltip(_activator, _args.Model)); + //Add(new ExecuteCommandShowTooltip(_activator, _args.Model)); Add(new ExecuteCommandShowKeywordHelp(_activator, _args)); var gotoMenu = Items.OfType().FirstOrDefault(i => i.Text.Equals(AtomicCommandFactory.GoTo)); From 94f28abfff193264e1988deed8e8bfd4185c8db0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 24 Jul 2025 14:52:59 +0100 Subject: [PATCH 039/142] cohorts ui --- ...dAssociateCohortIdentificationConfigurationWithProject.cs | 4 ++-- .../AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAssociateCohortIdentificationConfigurationWithProject.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAssociateCohortIdentificationConfigurationWithProject.cs index 23fe289151..35e3e007ea 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAssociateCohortIdentificationConfigurationWithProject.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAssociateCohortIdentificationConfigurationWithProject.cs @@ -9,6 +9,7 @@ using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.DataExport.Data; using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.Providers; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; @@ -28,8 +29,7 @@ public ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(IBasi if (!activator.CoreChildProvider.AllCohortIdentificationConfigurations.Any()) SetImpossible("There are no Cohort Identification Configurations yet"); - _existingAssociations = BasicActivator.RepositoryLocator.DataExportRepository - .GetAllObjects(); + _existingAssociations = ((DataExportChildProvider)activator.CoreChildProvider).AllProjectAssociatedCics; } public override string GetCommandHelp() => diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs index c838effd4b..804f82b2c7 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs @@ -24,8 +24,9 @@ public ExecuteCommandSetQueryCachingDatabase(IBasicActivateItems activator, Coho base(activator) { _cic = cic; - _caches = BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(static s => s.WasCreatedBy(new QueryCachingPatcher())).ToArray(); + //_caches = BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects() + // .Where(static s => s.WasCreatedBy(new QueryCachingPatcher())).ToArray(); + _caches = activator.CoreChildProvider.AllExternalServers.Where(es => es.WasCreatedBy(new QueryCachingPatcher())).ToArray(); if (!_caches.Any()) SetImpossible("There are no Query Caching databases set up"); } From d39978dcf95d97e1ddfa9cf39f2453f720673ce6 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 24 Jul 2025 15:05:10 +0100 Subject: [PATCH 040/142] more speed improvements --- CHANGELOG.md | 3 ++ .../CommandExecution/AtomicCommandFactory.cs | 32 ++++++++--------- .../CommandExecution/GoToCommandFactory.cs | 35 +++++++++---------- .../ExecuteCommandViewCommits.cs | 6 ---- Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs | 2 +- 5 files changed, 37 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1704a624f8..0010f55480 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.0.1] - Unreleased +- Improve Right-click perfromance + ## [9.0.0] - 2025-07-23 - Add ability to use Extraction Configuration ID in the naming scheme of the extract to database Pipeline component - Update Cohort Versioning Icons and Interface diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 798459b1bb..f330637c4e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -899,22 +899,22 @@ public IEnumerable CreateCommands(object o) }; } - //if (Is(o, out IMightBeDeprecated d)) - //{ - // yield return new ExecuteCommandDeprecate(_activator, new[] { d }, !d.IsDeprecated) - // { - // OverrideCommandName = d.IsDeprecated ? "Un Deprecate" : "Deprecate", - // SuggestedCategory = Deprecation, - // Weight = -99.7f - // }; - // yield return new ExecuteCommandReplacedBy(_activator, d, null) - // { - // PromptToPickReplacement = true, - // SuggestedCategory = Deprecation, - // Weight = -99.6f, - // OverrideCommandName = "Set Replaced By" - // }; - //} + if (Is(o, out IMightBeDeprecated d)) + { + yield return new ExecuteCommandDeprecate(_activator, new[] { d }, !d.IsDeprecated) + { + OverrideCommandName = d.IsDeprecated ? "Un Deprecate" : "Deprecate", + SuggestedCategory = Deprecation, + Weight = -99.7f + }; + yield return new ExecuteCommandReplacedBy(_activator, d, null) + { + PromptToPickReplacement = true, + SuggestedCategory = Deprecation, + Weight = -99.6f, + OverrideCommandName = "Set Replaced By" + }; + } if (Is(o, out CohortAggregateContainer cohortAggregateContainer)) { diff --git a/Rdmp.Core/CommandExecution/GoToCommandFactory.cs b/Rdmp.Core/CommandExecution/GoToCommandFactory.cs index bade8a3eb1..68facfc5cf 100644 --- a/Rdmp.Core/CommandExecution/GoToCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/GoToCommandFactory.cs @@ -44,29 +44,28 @@ public IEnumerable GetCommands(object forObject) if (Is(forObject, out IInjectKnown ii)) ii.ClearAllInjections(); - //if (Is(forObject, out IMapsDirectlyToDatabaseTable mt)) - //{ - // // Go to import / export definitions - // var export = _activator.RepositoryLocator.CatalogueRepository.GetReferencesTo(mt) - // .FirstOrDefault(); + if (Is(forObject, out IMapsDirectlyToDatabaseTable mt)) + { + // Go to import / export definitions + var export = _activator.CoreChildProvider.AllExports.FirstOrDefault(export => export.IsReferenceTo(mt)); - // if (export != null) - // yield return new ExecuteCommandShow(_activator, export, 0, true) - // { OverrideCommandName = "Show Export Definition" }; + if (export != null) + yield return new ExecuteCommandShow(_activator, export, 0, true) + { OverrideCommandName = "Show Export Definition" }; - // var import = _activator.RepositoryLocator.CatalogueRepository.GetReferencesTo(mt) - // .FirstOrDefault(); - // if (import != null) - // yield return new ExecuteCommandShow(_activator, import, 0) - // { OverrideCommandName = "Show Import Definition" }; + var import = _activator.RepositoryLocator.CatalogueRepository.GetReferencesTo(mt) + .FirstOrDefault(); + if (import != null) + yield return new ExecuteCommandShow(_activator, import, 0) + { OverrideCommandName = "Show Import Definition" }; - // if (SupportsReplacement(forObject)) - // yield return new ExecuteCommandShow(_activator, () => GetReplacementIfAny(mt)) - // { OverrideCommandName = "Replacement" }; + if (SupportsReplacement(forObject)) + yield return new ExecuteCommandShow(_activator, () => GetReplacementIfAny(mt)) + { OverrideCommandName = "Replacement" }; - // yield return new ExecuteCommandSimilar(_activator, mt, false) { GoTo = true }; - //} + yield return new ExecuteCommandSimilar(_activator, mt, false) { GoTo = true }; + } // cic => associated projects if (Is(forObject, out CohortIdentificationConfiguration cic)) diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCommits.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCommits.cs index 840fb969dd..76c0b18b60 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCommits.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCommits.cs @@ -24,12 +24,6 @@ public ExecuteCommandViewCommits(IActivateItems activator, IMapsDirectlyToDataba { _o = o; OverrideCommandName = "View History"; - - //if ( - // !activator.RepositoryLocator.CatalogueRepository - // .GetAllObjectsWhere(nameof(Memento.ReferencedObjectID), o.ID) - // .Any(m => m.IsReferenceTo(o))) - // SetImpossible("No commits have been made yet"); } public override Image GetImage(IIconProvider iconProvider) => iconProvider.GetImage(RDMPConcept.Commit); diff --git a/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs b/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs index 9689e8cbb0..d4715c4a8b 100644 --- a/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs +++ b/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs @@ -28,7 +28,7 @@ public DQEMenuItem(IActivateItems activator, Catalogue catalogue) : base(activat Image = activator.CoreIconProvider.GetImage(RDMPConcept.DQE).ImageToBitmap(); - Text = "Data Quality Engine...";//_dqeServer == null ? "Create DQE Database..." : "Data Quality Engine..."; + Text = "Data Quality Engine..."; } protected override void OnClick(EventArgs e) From fe389a1b2c2588d58afbf908612e1bd64461dc80 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 24 Jul 2025 15:07:53 +0100 Subject: [PATCH 041/142] tidy up --- CHANGELOG.md | 2 +- .../AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs | 1 - Rdmp.UI/Menus/RDMPContextMenuStrip.cs | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0010f55480..b26c0505f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [9.0.1] - Unreleased -- Improve Right-click perfromance +- Improve Right-click performance ## [9.0.0] - 2025-07-23 - Add ability to use Extraction Configuration ID in the naming scheme of the extract to database Pipeline component diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs index 518da26bfe..bf04ff3d31 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs @@ -7,7 +7,6 @@ using System; using System.Collections.Generic; using System.Linq; -using NPOI.OpenXmlFormats; using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.Curation.Data; using Rdmp.Core.Icons.IconProvision; diff --git a/Rdmp.UI/Menus/RDMPContextMenuStrip.cs b/Rdmp.UI/Menus/RDMPContextMenuStrip.cs index 5d58a16dab..8e2f76814e 100644 --- a/Rdmp.UI/Menus/RDMPContextMenuStrip.cs +++ b/Rdmp.UI/Menus/RDMPContextMenuStrip.cs @@ -165,11 +165,11 @@ public void AddCommonMenuItems(RDMPCollectionCommonFunctionality commonFunctiona if (_o is IMapsDirectlyToDatabaseTable m) Add(new ExecuteCommandViewCommits(_activator, m)); - ////ensure all submenus appear in the same place + //ensure all submenus appear in the same place foreach (var mi in _subMenuDictionary.Values) Items.Add(mi); - ////add plugin menu items + //add plugin menu items foreach (var plugin in _activator.PluginUserInterfaces) try { From 2b206d84e59f66c7ed5b9ae5d9518eb41d103225 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 08:38:02 +0100 Subject: [PATCH 042/142] fix us tests --- .../Curation/ProjectspecificCatalogueTests.cs | 14 +++++++++-- .../ExecuteCommandAddNewAggregateGraph.cs | 2 +- .../ExecuteCommandChangeExtractability.cs | 4 +-- ...cuteCommandMakeCatalogueProjectSpecific.cs | 21 ++++++++++------ ...MakeProjectSpecificCatalogueNormalAgain.cs | 25 ++++++++++--------- .../ExecuteCommandSetColumnSettingBase.cs | 2 +- .../Data/ProjectSpecificCatalogueManager.cs | 15 +++++------ ...cuteCommandViewCatalogueExtractionSqlUI.cs | 5 +--- 8 files changed, 48 insertions(+), 40 deletions(-) diff --git a/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs b/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs index cc679bcb66..c2d52760ea 100644 --- a/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs +++ b/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs @@ -1,4 +1,5 @@ using Microsoft.Data.SqlClient; +using NPOI.OpenXmlFormats.Spreadsheet; using NUnit.Framework; using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; @@ -33,10 +34,13 @@ public void SetupTests(int projectCount = 0, int projectSpecificCount = 0, int e _catalogue.SaveToDatabase(); var _t1 = new TableInfo(CatalogueRepository, "T1"); _t1.SaveToDatabase(); + _activator.Publish(_t1); var _c1 = new ColumnInfo(CatalogueRepository, "PrivateIdentifierA", "varchar(10)", _t1); _c1.SaveToDatabase(); + _activator.Publish(_c1); var _ci1 = new CatalogueItem(CatalogueRepository, _catalogue, "PrivateIdentifierA"); _ci1.SaveToDatabase(); + _activator.Publish(_ci1); var _extractionInfo1 = new ExtractionInformation(CatalogueRepository, _ci1, _c1, _c1.ToString()) { Order = 123, @@ -44,18 +48,24 @@ public void SetupTests(int projectCount = 0, int projectSpecificCount = 0, int e IsExtractionIdentifier = true }; _extractionInfo1.SaveToDatabase(); + _activator.Publish(_extractionInfo1); if (projectCount > 0) { _project1 = new Project(DataExportRepository, "Project1"); _project1.SaveToDatabase(); + _activator.Publish(_project1); + } if (projectCount > 1) { _project2 = new Project(DataExportRepository, "Project1"); _project2.SaveToDatabase(); + _activator.Publish(_project2); + } _eds1 = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", _catalogue.ID).FirstOrDefault() ?? new ExtractableDataSet(_activator.RepositoryLocator.DataExportRepository, _catalogue); _eds1.SaveToDatabase(); + _activator.Publish(_eds1); if (projectSpecificCount > 0) { var cmd = new ExecuteCommandMakeCatalogueProjectSpecific(_activator); @@ -93,14 +103,14 @@ private void MakeProjectSpecific(Catalogue catalogue, Project project, List var cmd = new ExecuteCommandMakeCatalogueProjectSpecific(_activator,catalogue,project,force); cmd.SetTarget(project); cmd.SetTarget(catalogue); - if (shouldThrow) Assert.That(ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific((DataExportChildProvider)_activator.CoreChildProvider,catalogue,project,projectIdsToIgnore??new List()), Is.False); + if (shouldThrow) Assert.That(ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(_activator.RepositoryLocator.DataExportRepository, catalogue,project,projectIdsToIgnore??new List()), Is.False); else Assert.DoesNotThrow(() => cmd.Execute()); } private void MakeNotProjectSpecific(Catalogue catalogue, ExtractableDataSet eds, Project project, bool shouldThrow = false) { var cmd = new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, catalogue, eds); - if (shouldThrow) Assert.That(ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(catalogue, eds,project), Is.False); + if (shouldThrow) Assert.That(ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(_activator.RepositoryLocator.DataExportRepository, catalogue, eds,project), Is.False); else Assert.DoesNotThrow(() => cmd.Execute()); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs index 6d996b414b..87e5be28e3 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs @@ -25,7 +25,7 @@ public ExecuteCommandAddNewAggregateGraph(IBasicActivateItems activator, Catalog { _catalogue = catalogue; _name = name; - if (_catalogue != null && activator.CoreChildProvider.AllExtractionInformations.Where(ei => ei.CatalogueItem.Catalogue_ID == catalogue.ID && ei.ExtractionCategory == ExtractionCategory.Any).All(ei => ei.ColumnInfo == null)) + if (_catalogue != null && _catalogue.GetAllExtractionInformation(ExtractionCategory.Any).All(ei => ei.ColumnInfo == null)) SetImpossible("Catalogue has no extractable columns"); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs index 7a05adb0da..6625c06ce1 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs @@ -26,9 +26,7 @@ public ExecuteCommandChangeExtractability(IBasicActivateItems activator, Catalog //calls db but not sure how to improve this _catalogue = catalogue; var dataExportChildProvider = (DataExportChildProvider)activator.CoreChildProvider; - var eds = dataExportChildProvider.ExtractableDataSets.Where(eds => eds.Catalogue_ID == catalogue.ID).ToList(); - var status = eds.Count == 0 ? new CatalogueExtractabilityStatus(false, false) : new CatalogueExtractabilityStatus(true, eds.Count > 1 ? true : eds.First().Projects.Any()); - + var status = _catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository); if (status == null) { SetImpossible( diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs index 0d5622f0cf..14d0fb39c0 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs @@ -24,13 +24,14 @@ public class ExecuteCommandMakeCatalogueProjectSpecific : BasicCommandExecution, private IProject _project; private List _existingProjectIDs; private readonly bool _force = false; + private bool _hasRanCatalogueValidation = false; private readonly IBasicActivateItems _activator; [UseWithObjectConstructor] public ExecuteCommandMakeCatalogueProjectSpecific(IBasicActivateItems itemActivator, ICatalogue catalogue, IProject project, [DemandsInitialization("Ignore Validation",DemandType.Unspecified,defaultValue:false)]bool force) : this(itemActivator) { - SetCatalogue(catalogue); + _catalogue = catalogue; _project = project; _force = force; _activator = itemActivator; @@ -49,6 +50,10 @@ public override void Execute() { if (_catalogue == null) SetCatalogue(SelectOne(BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList())); + if (!_hasRanCatalogueValidation) + { + SetCatalogue(_catalogue); + } if(_existingProjectIDs is null) GetExistingProjectIDs(); @@ -86,8 +91,8 @@ private List GetListOfValidProjects() { var dataExportChildProvider = ((DataExportChildProvider)_activator.CoreChildProvider); - var availableProjects = dataExportChildProvider.Projects.Where(p =>!dataExportChildProvider.ExtractableDataSetProjects.Where(edsp => edsp.Project_ID == p.ID).Select(edsp => edsp.DataSet.Catalogue).Contains(_catalogue)); - return availableProjects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(dataExportChildProvider, _catalogue, p, _existingProjectIDs)).ToList(); + var availableProjects = dataExportChildProvider.Projects.Where(p => !dataExportChildProvider.ExtractableDataSetProjects.Where(edsp => edsp.Project_ID == p.ID).Select(edsp => edsp.DataSet.Catalogue).Contains(_catalogue)); + return availableProjects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(_activator.RepositoryLocator.DataExportRepository, _catalogue, p, _existingProjectIDs)).ToList(); } private void GetExistingProjectIDs() @@ -108,10 +113,8 @@ private void SetCatalogue(ICatalogue catalogue) SetImpossible("Catalogue cannot be null"); return; } - var eds = dataExportChildProvider.ExtractableDataSets.Where(eds => eds.Catalogue_ID == catalogue.ID).ToList(); - var status = eds.Count == 0 ? new CatalogueExtractabilityStatus(false, false) : new CatalogueExtractabilityStatus(true, eds.Count > 1 ? true : eds.First().Projects.Any()); - - if (!GetListOfValidProjects().Any() && !_force) + var status = _catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository); + if (!GetListOfValidProjects().Any() && !_force) { SetImpossible("No valid Projects available"); } @@ -119,12 +122,14 @@ private void SetCatalogue(ICatalogue catalogue) if (!status.IsExtractable) SetImpossible("Catalogue must first be made Extractable"); - var ei = dataExportChildProvider.AllCatalogueItems.Where(ci => ci.Catalogue_ID == _catalogue.ID && ci.ExtractionInformation != null).Select(ci => ci.ExtractionInformation); + + var ei = _catalogue.GetAllExtractionInformation(ExtractionCategory.Any); if (!ei.Any()) SetImpossible("Catalogue has no extractable columns"); if (ei.Count(e => e.IsExtractionIdentifier) < 1) SetImpossible("Catalogue must have at least 1 IsExtractionIdentifier column"); + _hasRanCatalogueValidation = true; } } \ No newline at end of file diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs index 348649a090..8b6bf82bbb 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs @@ -19,7 +19,7 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public class ExecuteCommandMakeProjectSpecificCatalogueNormalAgain : BasicCommandExecution, IAtomicCommand { private Catalogue _catalogue; - private readonly List _extractableDataSets; + private List _extractableDataSets; private ExtractableDataSet _extractableDataSet; private Project _selectedProj; @@ -36,15 +36,6 @@ public ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(IBasicActivateItems SetImpossible("Data Export functionality is not available"); return; } - - _extractableDataSets = dataExportRepository.GetAllObjectsWithParent(catalogue).Where(eds => eds.Projects.Any() && eds.Projects.Select(p =>ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(catalogue, eds,p)).Contains(true)).ToList(); - if (!_extractableDataSets.Any()) - { - SetImpossible("Cannot make Catalogue Non-Project specific"); - return; - } - - } public override string GetCommandHelp() => @@ -52,12 +43,22 @@ public override string GetCommandHelp() => public override void Execute() { + var dataExportRepository = BasicActivator.RepositoryLocator.DataExportRepository; + base.Execute(); + _extractableDataSets = dataExportRepository.GetAllObjectsWithParent(_catalogue).Where(eds => eds.Projects.Any() && eds.Projects.Select(p => ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(dataExportRepository, _catalogue, eds, p)).Contains(true)).ToList(); + if (!_extractableDataSets.Any()) + { + SetImpossible("Cannot make Catalogue Non-Project specific"); + return; + } + + + if (_extractableDataSet is null) { - var dataExportRepository = BasicActivator.RepositoryLocator.DataExportRepository; - var projectIds = _extractableDataSets.SelectMany(eds => eds.Projects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(_catalogue,eds,p))).Select(p => p.ID); + var projectIds = _extractableDataSets.SelectMany(eds => eds.Projects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(dataExportRepository,_catalogue, eds,p))).Select(p => p.ID); _selectedProj = SelectOne(BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjectsInIDList(projectIds).ToList()); if (_selectedProj is null) return; _extractableDataSet = _extractableDataSets.FirstOrDefault(eds => eds.Projects.Select(p => p.ID).Contains(_selectedProj.ID)); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs index 3fae936b7d..e60460c850 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs @@ -76,7 +76,7 @@ public ExecuteCommandSetColumnSettingBase( } else { - _extractionInformations = dataExportChildProvider.AllCatalogueItems.Where(ci => ci.Catalogue_ID == catalogue.ID && ci.ExtractionInformation != null && ci.ExtractionInformation.ExtractionCategory == ExtractionCategory.Any).Select(ci => ci.ExtractionInformation).ToArray(); + _extractionInformations = _catalogue.GetAllExtractionInformation(ExtractionCategory.Any); if (_extractionInformations.Length == 0) { diff --git a/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs b/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs index 6512c0883c..42df873bf8 100644 --- a/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs +++ b/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs @@ -1,7 +1,6 @@ using NPOI.OpenXmlFormats.Spreadsheet; using Org.BouncyCastle.Crypto.Signers; using Rdmp.Core.Curation.Data; -using Rdmp.Core.Providers; using Rdmp.Core.Repositories; using Spectre.Console; using System; @@ -16,10 +15,9 @@ namespace Rdmp.Core.DataExport.Data static class ProjectSpecificCatalogueManager { - public static bool CanMakeCatalogueProjectSpecific(DataExportChildProvider childProvider, ICatalogue catalogue, IProject project, List projectIdsToIgnore) + public static bool CanMakeCatalogueProjectSpecific(IDataExportRepository dqeRepo, ICatalogue catalogue, IProject project, List projectIdsToIgnore) { - var foundeds = childProvider.ExtractableDataSets.Where(eds => eds.Catalogue_ID == catalogue.ID).ToList(); - var status = foundeds.Count == 0 ? new CatalogueExtractabilityStatus(false, false) : new CatalogueExtractabilityStatus(true, foundeds.Count > 1 ? true : foundeds.First().Projects.Any()); + var status = catalogue.GetExtractabilityStatus(dqeRepo); if (!status.IsExtractable) return false; @@ -29,7 +27,7 @@ public static bool CanMakeCatalogueProjectSpecific(DataExportChildProvider child return false; if (ei.Count(e => e.IsExtractionIdentifier) < 1) return false; - var edss = childProvider.ExtractableDataSets.Where(eds => eds.Catalogue_ID == catalogue.ID).ToList(); + var edss = dqeRepo.GetAllObjectsWithParent(catalogue); if (edss.Any(e => e.Projects.Select(p => p.ID).Contains(project.ID))) { //already project specific @@ -64,11 +62,11 @@ public static ExtractableDataSet MakeCatalogueProjectSpecific(IDataExportReposit return eds; } - public static bool CanMakeCatalogueNonProjectSpecific(ICatalogue catalogue, ExtractableDataSet extractableDataSet,IProject project) + public static bool CanMakeCatalogueNonProjectSpecific(IDataExportRepository dqeRepo, ICatalogue catalogue, ExtractableDataSet extractableDataSet, IProject project) { bool usedInExtraction = project.ExtractionConfigurations.Where(ec => ec.SelectedDataSets.Select(sds => sds.ExtractableDataSet).Contains(extractableDataSet)).Any(); if (!usedInExtraction) return true; - if(usedInExtraction && extractableDataSet.Projects.Count ==1) return true; + if (usedInExtraction && extractableDataSet.Projects.Count == 1) return true; return false; } @@ -84,7 +82,7 @@ public static void MakeCatalogueNonProjectSpecific(IDataExportRepository dqeRepo p.DeleteInDatabase(); } extractableDataSet.Projects.Remove(project); - + foreach (var ei in catalogue.GetAllExtractionInformation(ExtractionCategory.ProjectSpecific)) { ei.ExtractionCategory = ExtractionCategory.Core; @@ -93,4 +91,3 @@ public static void MakeCatalogueNonProjectSpecific(IDataExportRepository dqeRepo } } } - diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCatalogueExtractionSqlUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCatalogueExtractionSqlUI.cs index 09b3c95367..8d91f97714 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCatalogueExtractionSqlUI.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandViewCatalogueExtractionSqlUI.cs @@ -20,18 +20,15 @@ namespace Rdmp.UI.CommandExecution.AtomicCommands; public class ExecuteCommandViewCatalogueExtractionSqlUI : BasicUICommandExecution, IAtomicCommandWithTarget { private Catalogue _catalogue; - private IActivateItems _actvator; [UseWithObjectConstructor] public ExecuteCommandViewCatalogueExtractionSqlUI(IActivateItems activator, Catalogue catalogue) : this(activator) { _catalogue = catalogue; - _actvator = activator; } public ExecuteCommandViewCatalogueExtractionSqlUI(IActivateItems activator) : base(activator) { - _actvator = activator; } public override string GetCommandHelp() => @@ -45,7 +42,7 @@ public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) //if the catalogue has no extractable columns - if(_actvator.CoreChildProvider.AllCatalogueItems.Where(ci => ci.Catalogue_ID == _catalogue.ID).Where(ci => ci.ExtractionInformation != null).Any(ci => ci.ExtractionInformation.ExtractionCategory == ExtractionCategory.Any)) + if(!_catalogue.GetAllExtractionInformation(ExtractionCategory.Any).Any()) SetImpossible("Catalogue has no ExtractionInformations"); return this; From f69c43dc2e55a4c2bc2ceb241daa46f92a7d7273 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 09:02:48 +0100 Subject: [PATCH 043/142] tidy up tests --- .../AtomicCommands/ExecuteCommandSetColumnSettingBase.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs index e60460c850..6ad42f26ae 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs @@ -49,14 +49,13 @@ public ExecuteCommandSetColumnSettingBase( _commandName = commandName; _catalogue.ClearAllInjections(); _commandProperty = commandProperty; - var dataExportChildProvider = ((DataExportChildProvider)activator.CoreChildProvider); if (inConfiguration != null) { SetImpossibleIfReadonly(_inConfiguration); - var allEds = dataExportChildProvider.ExtractableDataSets.Where(eds => eds.ExtractionConfigurations.Contains(inConfiguration)); + var allEds = inConfiguration.GetAllExtractableDataSets(); var eds = allEds.FirstOrDefault(sds => sds.Catalogue_ID == _catalogue.ID); if (eds == null) { From 289284ccfeeb388e44f3ff47d9db8114d5358e46 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 09:15:18 +0100 Subject: [PATCH 044/142] tidy up code --- .../ExecuteCommandChangeExtractability.cs | 2 - ...cuteCommandMakeCatalogueProjectSpecific.cs | 1 - .../ExecuteCommandSetQueryCachingDatabase.cs | 2 - .../CommandExecution/GoToCommandFactory.cs | 1 - .../ExecuteCommandShowTooltip.cs | 49 ------------------- Rdmp.UI/Menus/RDMPContextMenuStrip.cs | 1 - 6 files changed, 56 deletions(-) delete mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandShowTooltip.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs index 6625c06ce1..24593de385 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs @@ -23,9 +23,7 @@ public sealed class ExecuteCommandChangeExtractability : BasicCommandExecution public ExecuteCommandChangeExtractability(IBasicActivateItems activator, Catalogue catalogue, bool? explicitExtractability = null) : base(activator) { - //calls db but not sure how to improve this _catalogue = catalogue; - var dataExportChildProvider = (DataExportChildProvider)activator.CoreChildProvider; var status = _catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository); if (status == null) { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs index 14d0fb39c0..8ba58e2e04 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs @@ -105,7 +105,6 @@ private void GetExistingProjectIDs() private void SetCatalogue(ICatalogue catalogue) { ResetImpossibleness(); - var dataExportChildProvider = ((DataExportChildProvider)_activator.CoreChildProvider); _catalogue = catalogue; GetExistingProjectIDs(); if (catalogue == null) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs index 804f82b2c7..67816f169c 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs @@ -24,8 +24,6 @@ public ExecuteCommandSetQueryCachingDatabase(IBasicActivateItems activator, Coho base(activator) { _cic = cic; - //_caches = BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects() - // .Where(static s => s.WasCreatedBy(new QueryCachingPatcher())).ToArray(); _caches = activator.CoreChildProvider.AllExternalServers.Where(es => es.WasCreatedBy(new QueryCachingPatcher())).ToArray(); if (!_caches.Any()) SetImpossible("There are no Query Caching databases set up"); diff --git a/Rdmp.Core/CommandExecution/GoToCommandFactory.cs b/Rdmp.Core/CommandExecution/GoToCommandFactory.cs index 68facfc5cf..4852d7589c 100644 --- a/Rdmp.Core/CommandExecution/GoToCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/GoToCommandFactory.cs @@ -192,7 +192,6 @@ public IEnumerable GetCommands(object forObject) ) { OverrideCommandName = "Usages (in Cohort Builder)" }; - //TODO yield return new ExecuteCommandShow(_activator, () => _activator.RepositoryLocator.DataExportRepository .GetAllObjectsWhere("ClonedFromExtractionFilter_ID", masterFilter.ID) diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandShowTooltip.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandShowTooltip.cs deleted file mode 100644 index 63c8e4245e..0000000000 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandShowTooltip.cs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) The University of Dundee 2018-2019 -// This file is part of the Research Data Management Platform (RDMP). -// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -// You should have received a copy of the GNU General Public License along with RDMP. If not, see . - -using System; -using Rdmp.Core.CommandExecution.AtomicCommands; -using Rdmp.Core.Icons.IconProvision; -using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; -using Rdmp.UI.Collections; -using Rdmp.UI.ItemActivation; -using SixLabors.ImageSharp; -using SixLabors.ImageSharp.PixelFormats; - -namespace Rdmp.UI.CommandExecution.AtomicCommands; - -public class ExecuteCommandShowTooltip : BasicUICommandExecution, IAtomicCommand -{ - private string _title; - private string _body; - private bool _isBad; - - public ExecuteCommandShowTooltip(IActivateItems activator, object o) : base(activator) - { - Weight = 100.5f; - //this REALLY queries the db - var hasOne = RDMPCollectionCommonFunctionality.GetToolTip(activator, o, out _title, out _body, out _isBad); - - if (!hasOne) - SetImpossible($"{o} does not have a tooltip/problem"); - } - - public override string GetCommandName() => _isBad ? "Show Problem" : "Show Tooltip"; - - public override Image GetImage(IIconProvider iconProvider) => - (Image)(_isBad ? Image.Load(FamFamFamIcons.flag_red) : iconProvider.GetImage(RDMPConcept.Help)); - - - public override void Execute() - { - base.Execute(); - - if (_isBad) - BasicActivator.ShowException(_title, new Exception(_body)); - else - BasicActivator.Show(_title, _body); - } -} \ No newline at end of file diff --git a/Rdmp.UI/Menus/RDMPContextMenuStrip.cs b/Rdmp.UI/Menus/RDMPContextMenuStrip.cs index 8e2f76814e..019ff4488f 100644 --- a/Rdmp.UI/Menus/RDMPContextMenuStrip.cs +++ b/Rdmp.UI/Menus/RDMPContextMenuStrip.cs @@ -207,7 +207,6 @@ public void AddCommonMenuItems(RDMPCollectionCommonFunctionality commonFunctiona if (databaseEntity != null) Add(new ExecuteCommandRefreshObject(_activator, databaseEntity), Keys.F5); - //Add(new ExecuteCommandShowTooltip(_activator, _args.Model)); Add(new ExecuteCommandShowKeywordHelp(_activator, _args)); var gotoMenu = Items.OfType().FirstOrDefault(i => i.Text.Equals(AtomicCommandFactory.GoTo)); From 11ce148b7f42214d6f53949ddcfdca5c68f79447 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 09:31:33 +0100 Subject: [PATCH 045/142] update version name --- SharedAssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 559a9fe198..6096b7fbb7 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: AssemblyVersion("9.0.0")] [assembly: AssemblyFileVersion("9.0.0")] -[assembly: AssemblyInformationalVersion("9.0.0")] +[assembly: AssemblyInformationalVersion("9.0.0-right-click-test")] From 10b7c50618e892d65f6c4e29cffdc0e52216b31c Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 09:33:44 +0100 Subject: [PATCH 046/142] fix test --- Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs b/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs index d0463360d8..1cc6d51ca7 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs @@ -31,7 +31,6 @@ private List allowedToBeIncompatible typeof(ExecuteCommandSetDataAccessContextForCredentials), typeof(ExecuteCommandActivate), typeof(ExecuteCommandCreateNewExternalDatabaseServer), - typeof(ExecuteCommandShowTooltip), typeof(ExecuteCommandShowKeywordHelp), typeof(ExecuteCommandCollapseChildNodes), typeof(ExecuteCommandExpandAllNodes), From 59395773d9caacb4b6399f2f0b8c308df9cae975 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 14:47:48 +0100 Subject: [PATCH 047/142] Potential fix for code scanning alert no. 12121: Virtual call in constructor or destructor Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs b/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs index d4715c4a8b..6d62501ac0 100644 --- a/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs +++ b/Rdmp.UI/Menus/MenuItems/DQEMenuItem.cs @@ -28,6 +28,11 @@ public DQEMenuItem(IActivateItems activator, Catalogue catalogue) : base(activat Image = activator.CoreIconProvider.GetImage(RDMPConcept.DQE).ImageToBitmap(); + InitializeText(); + } + + private void InitializeText() + { Text = "Data Quality Engine..."; } From ea43a8b5b2befe501512b038d023b0eaa782797c Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 14:49:00 +0100 Subject: [PATCH 048/142] update shared assembly --- SharedAssemblyInfo.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 6096b7fbb7..aa79dc3a33 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("9.0.0")] -[assembly: AssemblyFileVersion("9.0.0")] -[assembly: AssemblyInformationalVersion("9.0.0-right-click-test")] +[assembly: AssemblyVersion("9.0.1")] +[assembly: AssemblyFileVersion("9.0.1")] +[assembly: AssemblyInformationalVersion("9.0.1")] From bca043d90d24ed56d7457563b6d99f06c8e8f2a7 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Jul 2025 11:00:40 +0100 Subject: [PATCH 049/142] add handling for orphaned filters --- .../AtomicCommands/ExecuteCommandCreateNewFilter.cs | 2 +- SharedAssemblyInfo.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs index dc52a18a0e..2cc6f7281b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs @@ -65,7 +65,7 @@ public bool OfferCohortCatalogueFilters { var c = GetCatalogue(); - var filters = c.CatalogueRepository.GetAllObjects().Where(af => af.GetCatalogue().ID == c.ID); + var filters = c.CatalogueRepository.GetAllObjects().Where(af => { try { return af.GetCatalogue()?.ID == c.ID; } catch (Exception) { return false; } }); _offerCohortFilters = filters.ToArray(); offerCohortCatalogueFilters = value; } diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index aa79dc3a33..cbc5cf65dd 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: AssemblyVersion("9.0.1")] [assembly: AssemblyFileVersion("9.0.1")] -[assembly: AssemblyInformationalVersion("9.0.1")] +[assembly: AssemblyInformationalVersion("9.0.1-cohort-filter-fix")] From 9a056bb8a5a8cfb9f287eeff60f03723540ce7c0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Jul 2025 11:07:44 +0100 Subject: [PATCH 050/142] add changelog --- CHANGELOG.md | 1 + SharedAssemblyInfo.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b26c0505f2..bbd9e446f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.0.1] - Unreleased - Improve Right-click performance +- Fix bug where orphaned filters were causing issues with cohort configurations ## [9.0.0] - 2025-07-23 - Add ability to use Extraction Configuration ID in the naming scheme of the extract to database Pipeline component diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index cbc5cf65dd..aa79dc3a33 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: AssemblyVersion("9.0.1")] [assembly: AssemblyFileVersion("9.0.1")] -[assembly: AssemblyInformationalVersion("9.0.1-cohort-filter-fix")] +[assembly: AssemblyInformationalVersion("9.0.1")] From 69d73b480c038c6f9689091004a29c35b78bfac2 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 31 Jul 2025 07:30:19 +0100 Subject: [PATCH 051/142] revert filter finder --- .../CommandExecution/AtomicCommandFactory.cs | 5 ++--- .../ExecuteCommandCreateNewFilter.cs | 18 +----------------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index f330637c4e..177411678d 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -261,7 +261,6 @@ public IEnumerable CreateCommands(object o) yield return new ExecuteCommandCreateNewFilter(_activator, ac) { OfferCatalogueFilters = true, - OfferCohortCatalogueFilters=true, SuggestedCategory = Add, OverrideCommandName = "Existing Filter" }; @@ -331,7 +330,6 @@ public IEnumerable CreateCommands(object o) yield return new ExecuteCommandCreateNewFilter(_activator, container, null) { OfferCatalogueFilters = true, - OfferCohortCatalogueFilters = true, SuggestedCategory = Add, OverrideCommandName = "Existing Filter" }; @@ -696,7 +694,6 @@ public IEnumerable CreateCommands(object o) yield return new ExecuteCommandCreateNewFilter(_activator, sds) { OfferCatalogueFilters = true, - OfferCohortCatalogueFilters = true, OverrideCommandName = "Existing Filter (copy of)", SuggestedCategory = Add }; @@ -950,10 +947,12 @@ public IEnumerable CreateCommands(object o) } if (Is(o, out IDisableable disable)) + //todo this calls the db yield return new ExecuteCommandDisableOrEnable(_activator, disable); // If the root object is deletable offer deleting if (Is(o, out IDeleteable deletable)) + //todo this calls the db yield return new ExecuteCommandDelete(_activator, deletable) { SuggestedShortcut = "Delete" }; if (Is(o, out ReferenceOtherObjectDatabaseEntity reference)) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs index 2cc6f7281b..79420e1e30 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs @@ -34,8 +34,6 @@ public class ExecuteCommandCreateNewFilter : BasicCommandExecution, IAtomicComma private IFilter[] _offerFilters = []; private bool offerCatalogueFilters; - private IFilter[] _offerCohortFilters = []; - private bool offerCohortCatalogueFilters; @@ -58,20 +56,6 @@ public bool OfferCatalogueFilters } } - public bool OfferCohortCatalogueFilters - { - get => offerCohortCatalogueFilters; - set - { - var c = GetCatalogue(); - - var filters = c.CatalogueRepository.GetAllObjects().Where(af => { try { return af.GetCatalogue()?.ID == c.ID; } catch (Exception) { return false; } }); - _offerCohortFilters = filters.ToArray(); - offerCohortCatalogueFilters = value; - } - } - - private ExecuteCommandCreateNewFilter(IBasicActivateItems activator) : base(activator) { Weight = DEFAULT_WEIGHT; @@ -257,7 +241,7 @@ private void ImportExistingFilter(IContainer container) { var wizard = new FilterImportWizard(BasicActivator); - var filters = _offerFilters.Union(_offerCohortFilters).ToArray(); + var filters = _offerFilters; var import = wizard.ImportManyFromSelection(container, filters).ToArray(); foreach (var f in import) container.AddChild(f); From 494b2e89bdf064385f64c1190fab2d9391f640fa Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 31 Jul 2025 08:21:06 +0100 Subject: [PATCH 052/142] bump deps --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 2eb0a64515..d932ac78ce 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -22,7 +22,7 @@ - + From b469c4f2d0345f80be7692cb31405182df25c68b Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 31 Jul 2025 14:34:01 +0100 Subject: [PATCH 053/142] update changelog --- CHANGELOG.md | 2 +- rdmp-client.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbd9e446f6..58ac8c8cbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [9.0.1] - Unreleased +## [9.0.1] - 2025-07-31 - Improve Right-click performance - Fix bug where orphaned filters were causing issues with cohort configurations diff --git a/rdmp-client.xml b/rdmp-client.xml index 78c1ea3da3..3e8b773a9a 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -1,7 +1,7 @@ - 9.0.0.0 - https://github.com/HicServices/RDMP/releases/download/v9.0.0/rdmp-9.0.0-client.zip + 9.0.1.0 + https://github.com/HicServices/RDMP/releases/download/v9.0.1/rdmp-9.0.1-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true From 9266201bdfb70fce579e79f0aa8a3f96d1df68ff Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 20 Aug 2025 10:49:52 +0100 Subject: [PATCH 054/142] add remote sql executer --- CHANGELOG.md | 3 + .../Mutilators/RemoteServerSQLExecution.cs | 62 +++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 Rdmp.Core/DataLoad/Modules/Mutilators/RemoteServerSQLExecution.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 58ac8c8cbb..76bcc68f3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.0.2] - Unreleased +- Add Data Load component to allow SQL to be executed on an external database server + ## [9.0.1] - 2025-07-31 - Improve Right-click performance - Fix bug where orphaned filters were causing issues with cohort configurations diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RemoteServerSQLExecution.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RemoteServerSQLExecution.cs new file mode 100644 index 0000000000..9027c6b6a5 --- /dev/null +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RemoteServerSQLExecution.cs @@ -0,0 +1,62 @@ +using FAnsi.Discovery; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.DataLoad.Engine.Mutilators; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.DataLoad.Modules.Mutilators +{ + internal class RemoteServerSQLExecution : IPluginMutilateDataTables + { + + [DemandsInitialization("The Remote Database server to run this sql on")] + public ExternalDatabaseServer RemoteServer { get; set; } + + [DemandsInitialization("Run the following SQL when this component is run in the DLE", DemandType = DemandType.SQL, + Mandatory = true)] + public string Sql { get; set; } + + public void Check(ICheckNotifier notifier) + { + if (!RemoteServer.Exists()) + { + notifier.OnCheckPerformed(new CheckEventArgs("Remote Server unavailable", CheckResult.Fail)); + } + } + + public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage) + { + } + + public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener) + { + } + + public ExitCodeType Mutilate(IDataLoadJob job) + { + var db = RemoteServer.Discover(ReusableLibraryCode.DataAccess.DataAccessContext.DataLoad); + try + { + using (var conn = db.Server.GetConnection()) + { + conn.Open(); + var cmd = db.Server.GetCommand(Sql, conn); + cmd.ExecuteNonQuery(); + } + } + catch (Exception e) + { + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, e.Message)); + return ExitCodeType.Error; + } + return ExitCodeType.Success; + } + } +} From 52560b8c5dd9259f1fa06ac8c4ce94039032934d Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 5 Sep 2025 13:20:37 +0100 Subject: [PATCH 055/142] Task/rdmp 294 improve logging display (#2160) * expanding trees * add changelog * tidy up code * improve log filtering * tidy up imports * tidy up * add highlight * fix up checking * fix bad merge --- CHANGELOG.md | 1 + .../LoadEvents/LoadEventsTreeView.cs | 103 +++++++++++++++++- SharedAssemblyInfo.cs | 6 +- 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76bcc68f3c..3d4ac1eedc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.0.2] - Unreleased - Add Data Load component to allow SQL to be executed on an external database server +- Improve Extraction Log Viewer Filter ## [9.0.1] - 2025-07-31 - Improve Right-click performance diff --git a/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs b/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs index a20dc80d11..1bb71c0d3b 100644 --- a/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs +++ b/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs @@ -101,6 +101,7 @@ private void TbToFetchTextChanged(object sender, EventArgs e) private object olvDescription_AspectGetter(object rowObject) { + if (rowObject is null) return null; return rowObject switch { ArchivalDataLoadInfo adi => adi.ToString(), @@ -117,6 +118,7 @@ private object olvDescription_AspectGetter(object rowObject) private static object olvDate_AspectGetter(object rowObject) { + if (rowObject is null) return null; return rowObject switch { ArchivalDataLoadInfo adi => adi.StartTime, @@ -282,11 +284,110 @@ private void llLoading_LinkClicked(object sender, LinkLabelLinkClickedEventArgs AbortWorkers(); } + private class LogFilter : AbstractModelFilter + { + public LogFilter() + { + + } + + public LogFilter(ObjectListView olv) + { + this.ListView = olv; + } + + public LogFilter(ObjectListView olv, string text) + { + this.ListView = olv; + this.Text = text; + } + + public LogFilter(ObjectListView olv, string text, StringComparison comparison) + { + this.ListView = olv; + this.Text = text; + this.StringComparison = comparison; + } + + public string Text; + public StringComparison StringComparison = StringComparison.InvariantCultureIgnoreCase; + + protected ObjectListView ListView; + + private List acceptableChildren = []; + private List acceptableRoots = []; + + public override bool Filter(object modelObject) + { + if (this.ListView == null || String.IsNullOrEmpty(this.Text)) + return true; + + foreach (OLVColumn column in this.ListView.Columns) + { + if (column.IsVisible) + { + string cellText = column.GetStringValue(modelObject); + if (cellText.IndexOf(this.Text, this.StringComparison) != -1) + { + if (modelObject is ArchivalDataLoadInfo adli) + { + acceptableChildren.AddRange(adli.Progress.Select(p => p.ID)); + acceptableRoots.Add(adli.ID); + } + return true; + } + else if (modelObject is ArchivalProgressLog log) + { + if (acceptableChildren.Contains(log.ID)) + { + return true; + } + } + else if (modelObject is LoadEventsTreeView_Category lec) + { + if (acceptableRoots.Contains(lec.RunId)) + { + return true; + } + if (lec.Children is ArchivalProgressLog[] lapl) + { + if (lapl.Any(pl => column.GetStringValue(pl).IndexOf(this.Text, this.StringComparison) != -1)) return true; + } + if (lec.Children is ArchivalTableLoadInfo[] latli) + { + if (latli.Any(pl => column.GetStringValue(pl).IndexOf(this.Text, this.StringComparison) != -1)) return true; + } + if (lec.Children is ArchivalFatalError[] lafe) + { + if (lafe.Any(pl => column.GetStringValue(pl).IndexOf(this.Text, this.StringComparison) != -1)) return true; + } + } + else if (modelObject is ArchivalTableLoadInfo dli) + { + if (acceptableRoots.Contains(dli.Parent.ID)) + { + return true; + } + } + else if (modelObject is ArchivalDataLoadInfo adli) + { + if (adli.Progress.Any(p => column.GetStringValue(p).IndexOf(this.Text, this.StringComparison) != -1)) return true; + if (adli.TableLoadInfos.Any(p => column.GetStringValue(p).IndexOf(this.Text, this.StringComparison) != -1)) return true; + if (adli.Errors.Any(p => column.GetStringValue(p).IndexOf(this.Text, this.StringComparison) != -1)) return true; + } + } + } + + return false; + } + } public void ApplyFilter(string filter) { - treeView1.ModelFilter = new TextMatchFilter(treeView1, filter, StringComparison.CurrentCultureIgnoreCase); + + treeView1.ModelFilter = new LogFilter(treeView1, filter, StringComparison.CurrentCultureIgnoreCase); treeView1.UseFiltering = !string.IsNullOrWhiteSpace(filter); + treeView1.DefaultRenderer = new HighlightTextRenderer(new TextMatchFilter(treeView1, filter)); } private void treeView1_ColumnRightClick(object sender, CellRightClickEventArgs e) diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index aa79dc3a33..e9664cf2b3 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("9.0.1")] -[assembly: AssemblyFileVersion("9.0.1")] -[assembly: AssemblyInformationalVersion("9.0.1")] +[assembly: AssemblyVersion("9.0.2")] +[assembly: AssemblyFileVersion("9.0.2")] +[assembly: AssemblyInformationalVersion("9.0.2")] From f77576ea01e7116512723c33635bfa89d2cd26c4 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 5 Sep 2025 13:20:54 +0100 Subject: [PATCH 056/142] allow use of cic filters in extractions (#2221) --- CHANGELOG.md | 1 + .../ExecuteCommandCreateNewFilter.cs | 33 +++++++++++-------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d4ac1eedc..b9e6dcde0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [9.0.2] - Unreleased +- allow the use of Catalogue CIC filters in coresponding Extractions - Add Data Load component to allow SQL to be executed on an external database server - Improve Extraction Log Viewer Filter diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs index 79420e1e30..3755d0e694 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs @@ -10,8 +10,10 @@ using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; +using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.Curation.FilterImporting; using Rdmp.Core.Curation.FilterImporting.Construction; +using Rdmp.Core.DataExport.Data; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.Repositories.Construction; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; @@ -35,23 +37,11 @@ public class ExecuteCommandCreateNewFilter : BasicCommandExecution, IAtomicComma private IFilter[] _offerFilters = []; private bool offerCatalogueFilters; - - public bool OfferCatalogueFilters { get => offerCatalogueFilters; set { - if (value) - { - var c = GetCatalogue(); - _offerFilters = c?.GetAllFilters(); - - if (_offerFilters == null || !_offerFilters.Any()) - SetImpossible($"There are no Filters declared in Catalogue '{c?.ToString() ?? "NULL"}'"); - } - - offerCatalogueFilters = value; } } @@ -136,7 +126,6 @@ public ExecuteCommandCreateNewFilter(IBasicActivateItems activator, IRootFilterC _factory = host.GetFilterFactory(); _container = host.RootFilterContainer; _host = host; - if (_container == null && _host is AggregateConfiguration ac) { if (ac.Catalogue.IsApiCall()) @@ -204,6 +193,7 @@ public override void Execute() container = _host.RootFilterContainer; } + // if importing an existing filter instead of creating blank if (BasedOn != null) { @@ -212,6 +202,23 @@ public override void Execute() } else if (OfferCatalogueFilters) { + + var c = GetCatalogue(); + _offerFilters = c?.GetAllFilters(); + if (_host is SelectedDataSets sds) + { + var cohortId = sds.ExtractionConfiguration.Cohort.OriginID; + var cic = c.CatalogueRepository.GetObjectByID(cohortId); + if (cic != null) + { + var filters = cic.RootCohortAggregateContainer.GetAllAggregateConfigurationsRecursively().SelectMany(ac => ac.RootFilterContainer.GetAllFiltersIncludingInSubContainersRecursively()); + filters = filters.Where(f => f.GetCatalogue().ID == c.ID); + _offerFilters = _offerFilters.Concat(filters).ToArray(); + } + } + if (_offerFilters == null || !_offerFilters.Any()) + SetImpossible($"There are no Filters declared in Catalogue '{c?.ToString() ?? "NULL"}'"); + // we want user to make decision about what to import ImportExistingFilter(container); return; From bba73a19db0b56b5007287b7fd02a6d2ec4b71da Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 5 Sep 2025 13:21:29 +0100 Subject: [PATCH 057/142] RDMP-315 Flag Log view (#2222) * add flat log view * add missing file --- CHANGELOG.md | 1 + .../Settings/UserSettings.cs | 6 ++++ .../LoadEvents/LoadEventsTreeView.cs | 35 ++++++++++++++++--- .../SimpleDialogs/UserSettingsUI.Designer.cs | 20 +++++++++-- Rdmp.UI/SimpleDialogs/UserSettingsUI.cs | 1 + 5 files changed, 55 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9e6dcde0a..e810b41bd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [9.0.2] - Unreleased +- Add ability to view logs as a flat list. Default settings can be updated via user settings - allow the use of Catalogue CIC filters in coresponding Extractions - Add Data Load component to allow SQL to be executed on an external database server - Improve Extraction Log Viewer Filter diff --git a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs index 067881100a..f75fe71aed 100644 --- a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs +++ b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs @@ -195,6 +195,12 @@ public static string ExtractionWebhookUrl set => AppSettings.AddOrUpdateValue("ExtractionWebhookUrl", value); } + public static bool DefaultLogViewFlat + { + get => AppSettings.GetValueOrDefault("DefaultLogViewFlat", false); + set => AppSettings.AddOrUpdateValue("DefaultLogViewFlat", value); + } + #region Catalogue flag visibility settings public static bool ShowInternalCatalogues diff --git a/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs b/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs index 1bb71c0d3b..a69c6496a2 100644 --- a/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs +++ b/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs @@ -8,6 +8,7 @@ using System.Collections; using System.Collections.Generic; using System.ComponentModel; +using System.DirectoryServices.ActiveDirectory; using System.Drawing; using System.Linq; using System.Text; @@ -20,6 +21,7 @@ using Rdmp.Core.Logging; using Rdmp.Core.Logging.PastEvents; using Rdmp.Core.ReusableLibraryCode; +using Rdmp.Core.ReusableLibraryCode.Settings; using Rdmp.UI.Collections; using Rdmp.UI.ItemActivation; using Rdmp.UI.Menus.MenuItems; @@ -50,7 +52,9 @@ public partial class LoadEventsTreeView : RDMPUserControl, IObjectCollectionCont private readonly ToolStripButton _btnApplyFilter = new("Apply"); private readonly ToolStripTextBox _tbToFetch = new() { Text = "1000" }; private readonly ToolStripButton _btnFetch = new("Go"); + private readonly ToolStripButton _btnFlat = new("Flat View"); + private bool _flatView = UserSettings.DefaultLogViewFlat; private int _toFetch = 1000; @@ -58,7 +62,7 @@ public partial class LoadEventsTreeView : RDMPUserControl, IObjectCollectionCont public LoadEventsTreeView() { InitializeComponent(); - + if (_flatView) _btnFlat.Text = "Nested View"; _populateLoadHistory.DoWork += _populateLoadHistory_DoWork; _populateLoadHistory.WorkerSupportsCancellation = true; _populateLoadHistory.RunWorkerCompleted += _populateLoadHistory_RunWorkerCompleted; @@ -78,7 +82,7 @@ public LoadEventsTreeView() _btnApplyFilter.Click += (s, e) => ApplyFilter(_tbFilterBox.Text); _tbToFetch.TextChanged += TbToFetchTextChanged; _btnFetch.Click += (s, e) => PopulateLoadHistory(); - + _btnFlat.Click += (s,e) => ToggleView(); RDMPCollectionCommonFunctionality.SetupColumnTracking(treeView1, olvDescription, new Guid("6b09f39c-2b88-41ed-a396-42a2d2288952")); RDMPCollectionCommonFunctionality.SetupColumnTracking(treeView1, olvDate, @@ -130,9 +134,15 @@ private static object olvDate_AspectGetter(object rowObject) }; } - private static void treeView1_FormatRow(object sender, FormatRowEventArgs e) + private void treeView1_FormatRow(object sender, FormatRowEventArgs e) { // Only apply if it is a data load info thing + if (_flatView) + { + if (e.Model is ArchivalFatalError atli) + e.Item.ForeColor = Color.DarkOrange; + } + if (e.Model is not ArchivalDataLoadInfo dli) return; if (dli.HasErrors) @@ -208,7 +218,15 @@ private void _populateLoadHistory_RunWorkerCompleted(object sender, RunWorkerCom public void AddObjects(ArchivalDataLoadInfo[] archivalDataLoadInfos) { - treeView1.AddObjects(archivalDataLoadInfos); + if (_flatView) { + treeView1.AddObjects(archivalDataLoadInfos.SelectMany(adli => adli.TableLoadInfos).ToList()); + treeView1.AddObjects(archivalDataLoadInfos.SelectMany(adli => adli.Errors).ToList()); + treeView1.AddObjects(archivalDataLoadInfos.SelectMany(adli => adli.Progress).ToList()); + } + else + { + treeView1.AddObjects(archivalDataLoadInfos); + } } public void ClearObjects() @@ -243,6 +261,13 @@ private void _populateLoadHistory_DoWork(object sender, DoWorkEventArgs e) } } + private void ToggleView() + { + _flatView = !_flatView; + _btnFlat.Text = _flatView ?"Nested View": "Flat View"; + PopulateLoadHistory(); + } + private void PopulateLoadHistory() { //it's already doing it... @@ -507,7 +532,7 @@ public void SetCollection(IActivateItems activator, IPersistableObjectCollection CommonFunctionality.Add(new ToolStripLabel("Fetch:")); CommonFunctionality.Add(_tbToFetch); CommonFunctionality.Add(_btnFetch); - + CommonFunctionality.Add(_btnFlat); PopulateLoadHistory(); } } \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs b/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs index 4a8409fc1a..961a2a1718 100644 --- a/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs @@ -100,6 +100,7 @@ private void InitializeComponent() userSettingsToolTips = new System.Windows.Forms.ToolTip(components); tbFind = new System.Windows.Forms.TextBox(); label14 = new System.Windows.Forms.Label(); + cbFlatLogs = new System.Windows.Forms.CheckBox(); groupBox1.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)olvErrorCodes).BeginInit(); groupBox2.SuspendLayout(); @@ -454,7 +455,7 @@ private void InitializeComponent() // cbAutoRunSqlQueries // cbAutoRunSqlQueries.AutoSize = true; - cbAutoRunSqlQueries.Location = new System.Drawing.Point(4, 102); + cbAutoRunSqlQueries.Location = new System.Drawing.Point(7, 104); cbAutoRunSqlQueries.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); cbAutoRunSqlQueries.Name = "cbAutoRunSqlQueries"; cbAutoRunSqlQueries.Size = new System.Drawing.Size(138, 19); @@ -489,7 +490,7 @@ private void InitializeComponent() // cbNewFind // cbNewFind.AutoSize = true; - cbNewFind.Location = new System.Drawing.Point(6, 97); + cbNewFind.Location = new System.Drawing.Point(7, 97); cbNewFind.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); cbNewFind.Name = "cbNewFind"; cbNewFind.Size = new System.Drawing.Size(98, 19); @@ -631,7 +632,7 @@ private void InitializeComponent() cbPromptFilterRename.AutoSize = true; cbPromptFilterRename.Checked = true; cbPromptFilterRename.CheckState = System.Windows.Forms.CheckState.Checked; - cbPromptFilterRename.Location = new System.Drawing.Point(6, 89); + cbPromptFilterRename.Location = new System.Drawing.Point(7, 89); cbPromptFilterRename.Name = "cbPromptFilterRename"; cbPromptFilterRename.Size = new System.Drawing.Size(222, 19); cbPromptFilterRename.TabIndex = 4; @@ -660,6 +661,7 @@ private void InitializeComponent() // // groupBox7 // + groupBox7.Controls.Add(cbFlatLogs); groupBox7.Controls.Add(label16); groupBox7.Controls.Add(tbLogLocation); groupBox7.Controls.Add(label15); @@ -851,6 +853,17 @@ private void InitializeComponent() label14.TabIndex = 26; label14.Text = "Find Setting:"; // + // cbFlatLogs + // + cbFlatLogs.AutoSize = true; + cbFlatLogs.Location = new System.Drawing.Point(7, 257); + cbFlatLogs.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); + cbFlatLogs.Name = "cbFlatLogs"; + cbFlatLogs.Size = new System.Drawing.Size(140, 19); + cbFlatLogs.TabIndex = 29; + cbFlatLogs.Text = "Show Logs as Flat List"; + cbFlatLogs.UseVisualStyleBackColor = true; + // // UserSettingsFileUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); @@ -964,5 +977,6 @@ private void InitializeComponent() private System.Windows.Forms.TextBox tbWebhookUrl; private System.Windows.Forms.Label label18; private System.Windows.Forms.TextBox tbWebhookUsername; + private System.Windows.Forms.CheckBox cbFlatLogs; } } \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs b/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs index 75a35c30e1..84d81b5e2c 100644 --- a/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs +++ b/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs @@ -108,6 +108,7 @@ public UserSettingsFileUI(IActivateItems activator) RegisterCheckbox(cbUseAliasInsteadOfTransformInGroupByAggregateGraphs, nameof(UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs)); RegisterCheckbox(cbUseLocalFileSystem, nameof(UserSettings.UseLocalFileSystem)); + RegisterCheckbox(cbFlatLogs, nameof(UserSettings.DefaultLogViewFlat)); AddTooltip(label7, nameof(UserSettings.CreateDatabaseTimeout)); AddTooltip(tbCreateDatabaseTimeout, nameof(UserSettings.CreateDatabaseTimeout)); AddTooltip(label13, nameof(UserSettings.ArchiveTriggerTimeout)); From cb969384f7787e8d28faff3f3b7a72619cff5c1c Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 15 Sep 2025 08:34:42 +0100 Subject: [PATCH 058/142] add basic test --- .../DatasetVariableReportGeneratorTests.cs | 25 ++++++++++++++++++ .../DatasetVariableReportGenerator.cs | 26 +++++++------------ 2 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs diff --git a/Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs b/Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs new file mode 100644 index 0000000000..57ea50e013 --- /dev/null +++ b/Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs @@ -0,0 +1,25 @@ +using System.IO; +using NUnit.Framework; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.DataExport.DataExtraction.Pipeline; +using Rdmp.Core.Reports.ExtractionTime; +using Tests.Common.Scenarios; + +namespace Rdmp.Core.Tests.Reports.ExtractionTime +{ + internal class DatasetVariableReportGeneratorTests: TestsRequiringAnExtractionConfiguration + { + [Test] + public void Test_DatasetVariableReportGenerator_Creation() + { + this.Execute(out ExtractionPipelineUseCase pipelineUseCase,out var results); + var report = new DatasetVariableReportGenerator(pipelineUseCase); + report.GenerateDatasetVariableReport(); + var filename = Path.Combine( + pipelineUseCase.Destination.DirectoryPopulated.FullName, + $"{pipelineUseCase.Destination.GetFilename()}Variables.csv" + ); + Assert.That(File.Exists(filename)); + } + } +} diff --git a/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs index 89e84eb55f..4ba05c8de2 100644 --- a/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs +++ b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs @@ -1,34 +1,28 @@ using CommandLine; -using Rdmp.Core.CohortCommitting; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction.Pipeline; using Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; using Rdmp.Core.QueryBuilding; -using System; -using System.CodeDom.Compiler; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; -using System.Threading.Tasks; namespace Rdmp.Core.Reports.ExtractionTime { class DatasetVariableReportGenerator { - private ICatalogue _catalogue; - private ExtractionPipelineUseCase _executor; - private IExecuteDatasetExtractionDestination _destination; - private List _columnstoExtract; - private List _releaseSubs; + private readonly ICatalogue _catalogue; + private readonly IExecuteDatasetExtractionDestination _destination; + private readonly List _columnsToExtract; + private readonly List _releaseSubs; public DatasetVariableReportGenerator(ExtractionPipelineUseCase executer) { - _executor = executer; _destination = executer.Destination; _catalogue = executer.Source.Request.Catalogue; - _columnstoExtract = executer.Source.Request.ColumnsToExtract.Select(c => c.Cast()).ToList(); + _columnsToExtract = executer.Source.Request.ColumnsToExtract.Select(c => c.Cast()).ToList(); _releaseSubs = executer.Source.Request.ReleaseIdentifierSubstitutions; } @@ -36,7 +30,7 @@ public void GenerateDatasetVariableReport() { var csv = new StringBuilder(); WriteHeaders(csv); - foreach(var column in _columnstoExtract) + foreach(var column in _columnsToExtract) { WriteColumn(column, csv); } @@ -66,8 +60,8 @@ private void WriteColumn(ExtractableColumn column, StringBuilder sb) bool isIdentifier = catalogueItem.ExtractionInformation.IsExtractionIdentifier; var lookups = _catalogue.CatalogueRepository.GetAllObjectsWhere("ForeignKey_ID",column.ColumnInfo.ID); var lookupString = ""; - if (lookups.Any()) lookupString = string.Join(';', lookups.Select(l => LookupStringGenerator(l))); - sb.AppendLine($"\"{column.GetRuntimeName()}\",\"{column.ColumnInfo.Data_type}\",{isNull},\"{catalogueItem.Description}\",{isIdentifier},{lookups.Any()},{lookupString}"); + if (lookups.Length != 0) lookupString = string.Join(';', lookups.Select(l => LookupStringGenerator(l))); + sb.AppendLine($"\"{column.GetRuntimeName()}\",\"{column.ColumnInfo.Data_type}\",{isNull},\"{catalogueItem.Description}\",{isIdentifier},{lookups.Length != 0},{lookupString}"); } private void WriteReleaseSubs(ReleaseIdentifierSubstitution releaseIdentifierSubstitution,StringBuilder sb) @@ -76,12 +70,12 @@ private void WriteReleaseSubs(ReleaseIdentifierSubstitution releaseIdentifierSub var catalogueItem = _catalogue.CatalogueItems.Where(c => c.ColumnInfo_ID == column.ID).First(); var lookups = _catalogue.CatalogueRepository.GetAllObjectsWhere("ForeignKey_ID", column.ID); var lookupString = ""; - if (lookups.Any()) lookupString = string.Join(';', lookups.Select(l => LookupStringGenerator(l))); + if (lookups.Length != 0) lookupString = string.Join(';', lookups.Select(l => LookupStringGenerator(l))); sb.AppendLine($"\"{releaseIdentifierSubstitution.Alias}\",\"{column.Data_type}\",{false},\"{catalogueItem.Description}\",{releaseIdentifierSubstitution.IsExtractionIdentifier},{lookups.Any()},{lookupString}"); } - private void WriteHeaders(StringBuilder sb) + private static void WriteHeaders(StringBuilder sb) { sb.AppendLine("Variable Name, Type, Null possible(Y/N),Description,Identifier,HasLookups,Lookups"); } From 21204f331787c0bb5f879783f1981e39e377ff35 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 15 Sep 2025 09:21:03 +0100 Subject: [PATCH 059/142] expect file --- .../DataRelease/Potential/FlatFileReleasePotential.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/DataExport/DataRelease/Potential/FlatFileReleasePotential.cs b/Rdmp.Core/DataExport/DataRelease/Potential/FlatFileReleasePotential.cs index fb74c32fc7..57137b6ee3 100644 --- a/Rdmp.Core/DataExport/DataRelease/Potential/FlatFileReleasePotential.cs +++ b/Rdmp.Core/DataExport/DataRelease/Potential/FlatFileReleasePotential.cs @@ -49,6 +49,7 @@ private bool FilesAreMissing(IExtractionResults extractionResults) { ExtractFile = new FileInfo(extractionResults.DestinationDescription); var metadataFile = new FileInfo(extractionResults.DestinationDescription.Replace(".csv", ".docx")); + var variablesFile = new FileInfo(extractionResults.DestinationDescription.Replace(".csv","Variables.csv")); if (!ExtractFile.Exists) return true; //extract is missing @@ -61,7 +62,7 @@ private bool FilesAreMissing(IExtractionResults extractionResults) //see if there is any other pollution in the extract directory var unexpectedFile = ExtractFile.Directory.EnumerateFiles().FirstOrDefault(f => - !(f.Name.Equals(ExtractFile.Name) || f.Name.Equals(metadataFile.Name))); + !(f.Name.Equals(ExtractFile.Name) || f.Name.Equals(metadataFile.Name) || f.Name.Equals(variablesFile.Name))); if (unexpectedFile != null) throw new Exception( From 382670aa147e4fda42c6a10ed112a8cd66a4c6a4 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 15 Sep 2025 10:27:51 +0100 Subject: [PATCH 060/142] better handling --- .../Reports/ExtractionTime/DatasetVariableReportGenerator.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs index 4ba05c8de2..cf978a0fc6 100644 --- a/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs +++ b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs @@ -55,7 +55,9 @@ private string LookupStringGenerator(Lookup lookup) private void WriteColumn(ExtractableColumn column, StringBuilder sb) { - var catalogueItem = _catalogue.CatalogueItems.Where(c => c.ColumnInfo_ID == column.ColumnInfo.ID).First(); + if (_catalogue.CatalogueItems.Length == 0) return; + var catalogueItem = _catalogue.CatalogueItems.FirstOrDefault(c => c.ColumnInfo_ID == column.ColumnInfo.ID); + if (catalogueItem is null || catalogueItem.ExtractionInformation is null) return; bool isNull = !catalogueItem.ExtractionInformation.IsPrimaryKey; bool isIdentifier = catalogueItem.ExtractionInformation.IsExtractionIdentifier; var lookups = _catalogue.CatalogueRepository.GetAllObjectsWhere("ForeignKey_ID",column.ColumnInfo.ID); From 2df76b4dc738a1a3aba2ad8d57e944cc5458b0a9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 15 Sep 2025 16:19:43 +0100 Subject: [PATCH 061/142] tidy up code --- .../ExtractionTime/DatasetVariableReportGeneratorTests.cs | 2 +- .../Reports/ExtractionTime/DatasetVariableReportGenerator.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs b/Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs index 57ea50e013..2517b2e17e 100644 --- a/Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs +++ b/Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs @@ -15,7 +15,7 @@ public void Test_DatasetVariableReportGenerator_Creation() this.Execute(out ExtractionPipelineUseCase pipelineUseCase,out var results); var report = new DatasetVariableReportGenerator(pipelineUseCase); report.GenerateDatasetVariableReport(); - var filename = Path.Combine( + var filename = Path.Join( pipelineUseCase.Destination.DirectoryPopulated.FullName, $"{pipelineUseCase.Destination.GetFilename()}Variables.csv" ); diff --git a/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs index cf978a0fc6..e140e96c3f 100644 --- a/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs +++ b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs @@ -38,7 +38,7 @@ public void GenerateDatasetVariableReport() { WriteReleaseSubs(sub, csv); } - File.WriteAllText(new FileInfo(Path.Combine(_destination.DirectoryPopulated.FullName, + File.WriteAllText(new FileInfo(Path.Join(_destination.DirectoryPopulated.FullName, $"{_destination.GetFilename()}Variables.csv")).ToString(), csv.ToString()); } From d707f7d9e088caf787a70d2f72fd5b8d9a30e50d Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 26 Sep 2025 07:50:13 +0100 Subject: [PATCH 062/142] remove cold storage --- .../CustomMetadataSubstitutions.md | 1 - .../Providers/SearchablesMatchScorerTests.cs | 14 -------------- .../Reports/CustomMetadataReportTests.cs | 1 - Rdmp.Core/Curation/Data/Catalogue.cs | 11 ----------- Rdmp.Core/Curation/Data/ICatalogue.cs | 7 ------- .../Providers/CatalogueProblemProvider.cs | 2 +- Rdmp.Core/Providers/SearchablesMatchScorer.cs | 10 +++------- Rdmp.Core/Reports/DitaCatalogueExtractor.cs | 4 ++-- Rdmp.Core/Reports/GovernanceReport.cs | 2 +- .../Settings/UserSettings.cs | 6 ------ .../CatalogueCollectionFilterUI.Designer.cs | 14 -------------- .../Collections/CatalogueCollectionFilterUI.cs | 8 +------- .../CatalogueCollectionUI.Designer.cs | 18 ------------------ .../Filtering/CatalogueCollectionFilter.cs | 4 +--- Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs | 13 ------------- Rdmp.UI/MainFormUITabs/CatalogueUI.cs | 1 - .../CatalogueToDatasetLinkagePieChartUI.cs | 2 -- Rdmp.UI/PieCharts/GoodBadCataloguePieChart.cs | 2 -- ...GoodBadCataloguePieChartObjectCollection.cs | 6 ------ Rdmp.UI/SimpleDialogs/NewfindUI.cs | 2 -- .../SimpleDialogs/Reports/MetadataReportUI.cs | 2 +- Rdmp.UI/SimpleDialogs/SelectDialog.cs | 2 -- 22 files changed, 10 insertions(+), 122 deletions(-) diff --git a/Documentation/CodeTutorials/CustomMetadataSubstitutions.md b/Documentation/CodeTutorials/CustomMetadataSubstitutions.md index 4a4834f594..a5a992fa9c 100644 --- a/Documentation/CodeTutorials/CustomMetadataSubstitutions.md +++ b/Documentation/CodeTutorials/CustomMetadataSubstitutions.md @@ -111,7 +111,6 @@ When running with `oneFile` off (or inside a `$foreach $Catalogue` block) the fo |$Geographical_coverage|User provided | |$Granularity|User provided | |$ID|RDMP allocated number uniquely identifying this object instance in this RDMP installation| -|$IsColdStorageDataset|True or False flag, configurable in RDMP| |$IsDeprecated|True or False flag, configurable in RDMP| |$IsInternalDataset|True or False flag, configurable in RDMP| |$Last_revision_date|User provided| diff --git a/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs b/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs index c2a6ed9397..4ea3f7bb01 100644 --- a/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs +++ b/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs @@ -124,19 +124,6 @@ public void TestScoringCatalogueFlag_IsDeprecated(bool hasFlag, bool shouldShow, }, expectedResult); } - [TestCase(true, true, true)] - [TestCase(true, false, false)] - [TestCase(false, true, true)] - [TestCase(false, false, true)] - public void TestScoringCatalogueFlag_IsColdStorage(bool hasFlag, bool shouldShow, bool expectedResult) - { - TestScoringFlag((c, eds) => - { - c.IsColdStorageDataset = hasFlag; - UserSettings.ShowColdStorageCatalogues = shouldShow; - }, expectedResult); - } - [TestCase(true, true, true)] [TestCase(true, false, false)] [TestCase(false, true, true)] @@ -197,7 +184,6 @@ private void TestScoringFlag(Action setter, bool UserSettings.ShowNonExtractableCatalogues = false; UserSettings.ShowProjectSpecificCatalogues = false; UserSettings.ShowInternalCatalogues = false; - UserSettings.ShowColdStorageCatalogues = false; var c = WhenIHaveA(); c.Name = "Bunny"; diff --git a/Rdmp.Core.Tests/Reports/CustomMetadataReportTests.cs b/Rdmp.Core.Tests/Reports/CustomMetadataReportTests.cs index bf2a9a163a..8223b48985 100644 --- a/Rdmp.Core.Tests/Reports/CustomMetadataReportTests.cs +++ b/Rdmp.Core.Tests/Reports/CustomMetadataReportTests.cs @@ -1384,7 +1384,6 @@ public void TestAllSubs_Catalogue() $Geographical_coverage $Granularity $ID -$IsColdStorageDataset $IsDeprecated $IsInternalDataset $Last_revision_date diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 8c3da32b74..a7a58ab621 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -85,7 +85,6 @@ public sealed class Catalogue : DatabaseEntity, IComparable, ICatalogue, IInject private int? _pivotCategoryExtractionInformationID; private bool _isDeprecated; private bool _isInternalDataset; - private bool _isColdStorageDataset; private int? _liveLoggingServerID; private string _shortDescription; @@ -489,15 +488,6 @@ public bool IsInternalDataset set => SetField(ref _isInternalDataset, value); } - /// - [DoNotExtractProperty] - [DoNotImportDescriptions] - public bool IsColdStorageDataset - { - get => _isColdStorageDataset; - set => SetField(ref _isColdStorageDataset, value); - } - /// [Relationship(typeof(ExternalDatabaseServer), RelationshipType.LocalReference)] [DoNotExtractProperty] @@ -1104,7 +1094,6 @@ internal Catalogue(ICatalogueRepository repository, DbDataReader r) Source_URL = ParseUrl(r, "Source_URL"); IsDeprecated = (bool)r["IsDeprecated"]; IsInternalDataset = (bool)r["IsInternalDataset"]; - IsColdStorageDataset = (bool)r["IsColdStorageDataset"]; Folder = r["Folder"].ToString(); diff --git a/Rdmp.Core/Curation/Data/ICatalogue.cs b/Rdmp.Core/Curation/Data/ICatalogue.cs index af2b8c5b24..4e6632b3fe 100644 --- a/Rdmp.Core/Curation/Data/ICatalogue.cs +++ b/Rdmp.Core/Curation/Data/ICatalogue.cs @@ -77,13 +77,6 @@ public interface ICatalogue : IHasDependencies, IHasQuerySyntaxHelper, INamed, I /// bool IsInternalDataset { get; set; } - /// - /// Bit flag indicating whether the Catalogue is a seldom used dataset that should be hidden by default. Use this if you are importing lots of researcher - /// datasets for cohort generation / extraction but don't want them to clog up your user interface. - /// - bool IsColdStorageDataset { get; set; } - - /// /// User specified free text field. Not used for anything by RDMP. /// diff --git a/Rdmp.Core/Providers/CatalogueProblemProvider.cs b/Rdmp.Core/Providers/CatalogueProblemProvider.cs index 3c4e5217ea..2f6731e291 100644 --- a/Rdmp.Core/Providers/CatalogueProblemProvider.cs +++ b/Rdmp.Core/Providers/CatalogueProblemProvider.cs @@ -232,7 +232,7 @@ private string DescribeProblem(AllGovernanceNode allGovernanceNode) } var expiredCatalogues = expiredCatalogueIds.Select(id => _childProvider.AllCataloguesDictionary[id]) - .Where(c => !c.IsDeprecated /* || c.IsColdStorage || c.IsInternal*/).ToArray(); + .Where(c => !c.IsDeprecated /* || c.IsInternal*/).ToArray(); if (expiredCatalogues.Any()) return diff --git a/Rdmp.Core/Providers/SearchablesMatchScorer.cs b/Rdmp.Core/Providers/SearchablesMatchScorer.cs index 6f7183cba5..33229b67ee 100644 --- a/Rdmp.Core/Providers/SearchablesMatchScorer.cs +++ b/Rdmp.Core/Providers/SearchablesMatchScorer.cs @@ -34,7 +34,6 @@ public class SearchablesMatchScorer private readonly bool _scoreZeroForCohortAggregateContainers; private bool _showInternalCatalogues = true; private bool _showDeprecatedCatalogues = true; - private bool _showColdStorageCatalogues = true; private bool _showProjectSpecificCatalogues = true; private bool _showNonExtractableCatalogues = true; @@ -175,7 +174,6 @@ private void SetupRespectUserSettings() { _showInternalCatalogues = !RespectUserSettings || UserSettings.ShowInternalCatalogues; _showDeprecatedCatalogues = !RespectUserSettings || UserSettings.ShowDeprecatedCatalogues; - _showColdStorageCatalogues = !RespectUserSettings || UserSettings.ShowColdStorageCatalogues; _showProjectSpecificCatalogues = !RespectUserSettings || UserSettings.ShowProjectSpecificCatalogues; _showNonExtractableCatalogues = !RespectUserSettings || UserSettings.ShowNonExtractableCatalogues; } @@ -277,7 +275,7 @@ private int _ScoreMatches(KeyValuePair /// private bool ScoreZeroBecauseOfUserSettings(KeyValuePair kvp) => - !Filter(kvp.Key, kvp.Value, _showInternalCatalogues, _showDeprecatedCatalogues, _showColdStorageCatalogues, + !Filter(kvp.Key, kvp.Value, _showInternalCatalogues, _showDeprecatedCatalogues, _showProjectSpecificCatalogues, _showNonExtractableCatalogues); private static Catalogue GetCatalogueIfAnyInDescendancy( @@ -326,12 +324,11 @@ private static int MatchCount(List regexes, string str) /// /// /// - /// /// /// /// True if the item should be shown to the user based on filters public static bool Filter(object modelObject, DescendancyList descendancy, bool includeInternal, - bool includeDeprecated, bool includeColdStorage, bool includeProjectSpecific, bool includeNonExtractable) + bool includeDeprecated, bool includeProjectSpecific, bool includeNonExtractable) { //doesn't relate to us... if (modelObject is not ICatalogue cata) @@ -352,9 +349,8 @@ public static bool Filter(object modelObject, DescendancyList descendancy, bool var isExtractable = cata.GetExtractabilityStatus(null) != null && cata.GetExtractabilityStatus(null).IsExtractable; - return (isExtractable && !cata.IsColdStorageDataset && !cata.IsDeprecated && !cata.IsInternalDataset && + return (isExtractable && !cata.IsDeprecated && !cata.IsInternalDataset && !isProjectSpecific) || - (includeColdStorage && cata.IsColdStorageDataset) || (includeDeprecated && cata.IsDeprecated) || (includeInternal && cata.IsInternalDataset) || (includeProjectSpecific && isProjectSpecific) || diff --git a/Rdmp.Core/Reports/DitaCatalogueExtractor.cs b/Rdmp.Core/Reports/DitaCatalogueExtractor.cs index b615ebdf07..6e09386165 100644 --- a/Rdmp.Core/Reports/DitaCatalogueExtractor.cs +++ b/Rdmp.Core/Reports/DitaCatalogueExtractor.cs @@ -72,7 +72,7 @@ public void Extract(IDataLoadEventListener listener) //get all the catalogues then sort them alphabetically var catas = new List(_repository.GetAllObjects() - .Where(c => !(c.IsDeprecated || c.IsInternalDataset || c.IsColdStorageDataset))); + .Where(c => !(c.IsDeprecated || c.IsInternalDataset))); catas.Sort(); var sw = Stopwatch.StartNew(); @@ -270,7 +270,7 @@ private void GenerateIntroductionFile(string filename) /// public void Check(ICheckNotifier notifier) { - var catas = _repository.GetAllObjects().Where(c => !c.IsInternalDataset && !c.IsColdStorageDataset) + var catas = _repository.GetAllObjects().Where(c => !c.IsInternalDataset) .ToArray(); //Catalogues with no acronyms diff --git a/Rdmp.Core/Reports/GovernanceReport.cs b/Rdmp.Core/Reports/GovernanceReport.cs index d632a538c3..9d0c271bfd 100644 --- a/Rdmp.Core/Reports/GovernanceReport.cs +++ b/Rdmp.Core/Reports/GovernanceReport.cs @@ -61,7 +61,7 @@ public void GenerateReport() .Where(c => c.CatalogueItems.Any(ci => ci.ExtractionInformation != null)) .OrderBy(c => c.Name)) { - if (catalogue.IsDeprecated || catalogue.IsColdStorageDataset || catalogue.IsInternalDataset) + if (catalogue.IsDeprecated || catalogue.IsInternalDataset) continue; //get active governances diff --git a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs index f75fe71aed..4e3f067026 100644 --- a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs +++ b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs @@ -215,12 +215,6 @@ public static bool ShowDeprecatedCatalogues set => AppSettings.AddOrUpdateValue("ShowDeprecatedCatalogues", value); } - public static bool ShowColdStorageCatalogues - { - get => AppSettings.GetValueOrDefault("ShowColdStorageCatalogues", true); - set => AppSettings.AddOrUpdateValue("ShowColdStorageCatalogues", value); - } - public static bool ShowProjectSpecificCatalogues { get => AppSettings.GetValueOrDefault("ShowProjectSpecificCatalogues", true); diff --git a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs index 0abec794e8..99be71cb91 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs @@ -28,7 +28,6 @@ protected override void Dispose(bool disposing) /// private void InitializeComponent() { - this.cbShowColdStorage = new System.Windows.Forms.CheckBox(); this.cbShowDeprecated = new System.Windows.Forms.CheckBox(); this.cbShowInternal = new System.Windows.Forms.CheckBox(); this.cbProjectSpecific = new System.Windows.Forms.CheckBox(); @@ -37,17 +36,6 @@ private void InitializeComponent() this.flowLayoutPanel1.SuspendLayout(); this.SuspendLayout(); // - // cbShowColdStorage - // - this.cbShowColdStorage.AutoSize = true; - this.cbShowColdStorage.Location = new System.Drawing.Point(3, 3); - this.cbShowColdStorage.Name = "cbShowColdStorage"; - this.cbShowColdStorage.Size = new System.Drawing.Size(87, 17); - this.cbShowColdStorage.TabIndex = 5; - this.cbShowColdStorage.Text = "Cold Storage"; - this.cbShowColdStorage.UseVisualStyleBackColor = true; - this.cbShowColdStorage.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); - // // cbShowDeprecated // this.cbShowDeprecated.AutoSize = true; @@ -94,7 +82,6 @@ private void InitializeComponent() // // flowLayoutPanel1 // - this.flowLayoutPanel1.Controls.Add(this.cbShowColdStorage); this.flowLayoutPanel1.Controls.Add(this.cbShowInternal); this.flowLayoutPanel1.Controls.Add(this.cbShowDeprecated); this.flowLayoutPanel1.Controls.Add(this.cbProjectSpecific); @@ -118,7 +105,6 @@ private void InitializeComponent() #endregion - private System.Windows.Forms.CheckBox cbShowColdStorage; private System.Windows.Forms.CheckBox cbShowDeprecated; private System.Windows.Forms.CheckBox cbShowInternal; private System.Windows.Forms.CheckBox cbProjectSpecific; diff --git a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs index 5464cf7c63..cf4591ddb8 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs @@ -21,7 +21,6 @@ public CatalogueCollectionFilterUI() cbShowInternal.Checked = UserSettings.ShowInternalCatalogues; cbShowDeprecated.Checked = UserSettings.ShowDeprecatedCatalogues; - cbShowColdStorage.Checked = UserSettings.ShowColdStorageCatalogues; cbProjectSpecific.Checked = UserSettings.ShowProjectSpecificCatalogues; cbShowNonExtractable.Checked = UserSettings.ShowNonExtractableCatalogues; @@ -37,7 +36,6 @@ private void OnCheckboxChanged(object sender, EventArgs e) UserSettings.ShowInternalCatalogues = cbShowInternal.Checked; UserSettings.ShowDeprecatedCatalogues = cbShowDeprecated.Checked; - UserSettings.ShowColdStorageCatalogues = cbShowColdStorage.Checked; UserSettings.ShowProjectSpecificCatalogues = cbProjectSpecific.Checked; UserSettings.ShowNonExtractableCatalogues = cbShowNonExtractable.Checked; @@ -46,10 +44,9 @@ private void OnCheckboxChanged(object sender, EventArgs e) public void EnsureVisible(Catalogue c) { - if (c.IsColdStorageDataset || c.IsDeprecated || c.IsInternalDataset) + if ( c.IsDeprecated || c.IsInternalDataset) { //trouble is our flags might be hiding it so make sure it is visible - cbShowColdStorage.Checked = cbShowColdStorage.Checked || c.IsColdStorageDataset; cbShowDeprecated.Checked = cbShowDeprecated.Checked || c.IsDeprecated; cbShowInternal.Checked = cbShowInternal.Checked || c.IsInternalDataset; } @@ -72,9 +69,6 @@ public void CheckForChanges() if (cbShowDeprecated.Checked != UserSettings.ShowDeprecatedCatalogues) cbShowDeprecated.Checked = UserSettings.ShowDeprecatedCatalogues; - if (cbShowColdStorage.Checked != UserSettings.ShowColdStorageCatalogues) - cbShowColdStorage.Checked = UserSettings.ShowColdStorageCatalogues; - if (cbProjectSpecific.Checked != UserSettings.ShowProjectSpecificCatalogues) cbProjectSpecific.Checked = UserSettings.ShowProjectSpecificCatalogues; diff --git a/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs b/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs index f5393fc83a..fddb302018 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs @@ -29,12 +29,10 @@ private void InitializeComponent() olvFilters = new OLVColumn(); olvOrder = new OLVColumn(); imageList_RightClickIcons = new ImageList(components); - gbColdStorage = new GroupBox(); catalogueCollectionFilterUI1 = new CatalogueCollectionFilterUI(); panel2 = new Panel(); tbFilter = new TextBox(); ((ISupportInitialize)tlvCatalogues).BeginInit(); - gbColdStorage.SuspendLayout(); panel2.SuspendLayout(); SuspendLayout(); // @@ -90,18 +88,6 @@ private void InitializeComponent() imageList_RightClickIcons.Images.SetKeyName(3, "LOG"); imageList_RightClickIcons.Images.SetKeyName(4, "aggregates.png"); // - // gbColdStorage - // - gbColdStorage.Controls.Add(tbFilter); - gbColdStorage.Controls.Add(catalogueCollectionFilterUI1); - gbColdStorage.Dock = DockStyle.Bottom; - gbColdStorage.Location = new System.Drawing.Point(0, 414); - gbColdStorage.Name = "gbColdStorage"; - gbColdStorage.Size = new System.Drawing.Size(500, 65); - gbColdStorage.TabIndex = 1; - gbColdStorage.TabStop = false; - gbColdStorage.Text = "Show"; - // // catalogueCollectionFilterUI1 // catalogueCollectionFilterUI1.Dock = DockStyle.Fill; @@ -113,7 +99,6 @@ private void InitializeComponent() // panel2 // panel2.Controls.Add(tlvCatalogues); - panel2.Controls.Add(gbColdStorage); panel2.Dock = DockStyle.Fill; panel2.Location = new System.Drawing.Point(0, 0); panel2.Name = "panel2"; @@ -133,8 +118,6 @@ private void InitializeComponent() Name = "CatalogueCollectionUI"; Size = new System.Drawing.Size(500, 479); ((ISupportInitialize)tlvCatalogues).EndInit(); - gbColdStorage.ResumeLayout(false); - gbColdStorage.PerformLayout(); panel2.ResumeLayout(false); ResumeLayout(false); } @@ -145,7 +128,6 @@ private void InitializeComponent() private ImageList imageList_RightClickIcons; private OLVColumn olvColumn1; private OLVColumn olvFilters; - private GroupBox gbColdStorage; private OLVColumn olvOrder; private CatalogueCollectionFilterUI catalogueCollectionFilterUI1; private Panel panel2; diff --git a/Rdmp.UI/Collections/Providers/Filtering/CatalogueCollectionFilter.cs b/Rdmp.UI/Collections/Providers/Filtering/CatalogueCollectionFilter.cs index d1e6804a7d..067949a8f9 100644 --- a/Rdmp.UI/Collections/Providers/Filtering/CatalogueCollectionFilter.cs +++ b/Rdmp.UI/Collections/Providers/Filtering/CatalogueCollectionFilter.cs @@ -20,7 +20,6 @@ public class CatalogueCollectionFilter : IModelFilter public ICoreChildProvider ChildProvider { get; set; } private readonly bool _isInternal; private readonly bool _isDeprecated; - private readonly bool _isColdStorage; private readonly bool _isProjectSpecific; private readonly bool _isNonExtractable; @@ -29,12 +28,11 @@ public CatalogueCollectionFilter(ICoreChildProvider childProvider) ChildProvider = childProvider; _isInternal = UserSettings.ShowInternalCatalogues; _isDeprecated = UserSettings.ShowDeprecatedCatalogues; - _isColdStorage = UserSettings.ShowColdStorageCatalogues; _isProjectSpecific = UserSettings.ShowProjectSpecificCatalogues; _isNonExtractable = UserSettings.ShowNonExtractableCatalogues; } public bool Filter(object modelObject) => SearchablesMatchScorer.Filter(modelObject, - ChildProvider.GetDescendancyListIfAnyFor(modelObject), _isInternal, _isDeprecated, _isColdStorage, + ChildProvider.GetDescendancyListIfAnyFor(modelObject), _isInternal, _isDeprecated, _isProjectSpecific, _isNonExtractable); } \ No newline at end of file diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs index 9ce5b6b790..62abb71b24 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs @@ -33,7 +33,6 @@ protected override void Dispose(bool disposing) private void InitializeComponent() { splitContainer1 = new SplitContainer(); - cbColdStorage = new CheckBox(); cbInternal = new CheckBox(); cbDeprecated = new CheckBox(); editableFolder = new SimpleControls.EditableLabelUI(); @@ -177,7 +176,6 @@ private void InitializeComponent() // // splitContainer1.Panel1 // - splitContainer1.Panel1.Controls.Add(cbColdStorage); splitContainer1.Panel1.Controls.Add(cbInternal); splitContainer1.Panel1.Controls.Add(cbDeprecated); splitContainer1.Panel1.Controls.Add(editableFolder); @@ -191,16 +189,6 @@ private void InitializeComponent() splitContainer1.SplitterDistance = 158; splitContainer1.TabIndex = 0; // - // cbColdStorage - // - cbColdStorage.AutoSize = true; - cbColdStorage.Location = new System.Drawing.Point(695, 75); - cbColdStorage.Name = "cbColdStorage"; - cbColdStorage.Size = new System.Drawing.Size(94, 19); - cbColdStorage.TabIndex = 5; - cbColdStorage.Text = "Cold Storage"; - cbColdStorage.UseVisualStyleBackColor = true; - // // cbInternal // cbInternal.AutoSize = true; @@ -1177,7 +1165,6 @@ private void InitializeComponent() private TicketingControlUI ticketingControl1; private SimpleControls.EditableLabelUI editableFolder; private SimpleControls.EditableLabelUI editableCatalogueName; - private CheckBox cbColdStorage; private CheckBox cbInternal; private CheckBox cbDeprecated; private TabControl tabControl1; diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs index 16b3bf53d5..6a79f879bd 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs @@ -123,7 +123,6 @@ public override void SetDatabaseObject(IActivateItems activator, Catalogue datab protected override void SetBindings(BinderWithErrorProviderFactory rules, Catalogue databaseObject) { base.SetBindings(rules, databaseObject); - Bind(cbColdStorage, "Checked", "IsColdStorageDataset", c => c.IsColdStorageDataset); Bind(cbDeprecated, "Checked", "IsDeprecated", c => c.IsDeprecated); Bind(cbInternal, "Checked", "IsInternalDataset", c => c.IsInternalDataset); Bind(editableCatalogueName, "TextValue", "Name", c => c.Name); diff --git a/Rdmp.UI/PieCharts/CatalogueToDatasetLinkagePieChartUI.cs b/Rdmp.UI/PieCharts/CatalogueToDatasetLinkagePieChartUI.cs index e08d7786ae..598900ced4 100644 --- a/Rdmp.UI/PieCharts/CatalogueToDatasetLinkagePieChartUI.cs +++ b/Rdmp.UI/PieCharts/CatalogueToDatasetLinkagePieChartUI.cs @@ -73,8 +73,6 @@ private void SetupFlags() AddFlag("Deprecated Catalogues", c => c.IncludeDeprecatedCatalogues, (c, r) => c.IncludeDeprecatedCatalogues = r); AddFlag("Internal Catalogues", c => c.IncludeInternalCatalogues, (c, r) => c.IncludeInternalCatalogues = r); - AddFlag("Cold Storage Catalogues", c => c.IncludeColdStorageCatalogues, - (c, r) => c.IncludeColdStorageCatalogues = r); AddFlag("Project Specific Catalogues", c => c.IncludeProjectSpecificCatalogues, (c, r) => c.IncludeProjectSpecificCatalogues = r); diff --git a/Rdmp.UI/PieCharts/GoodBadCataloguePieChart.cs b/Rdmp.UI/PieCharts/GoodBadCataloguePieChart.cs index 0b126d190f..bc88a6a0ef 100644 --- a/Rdmp.UI/PieCharts/GoodBadCataloguePieChart.cs +++ b/Rdmp.UI/PieCharts/GoodBadCataloguePieChart.cs @@ -73,8 +73,6 @@ private void SetupFlags() AddFlag("Deprecated Catalogues", c => c.IncludeDeprecatedCatalogues, (c, r) => c.IncludeDeprecatedCatalogues = r); AddFlag("Internal Catalogues", c => c.IncludeInternalCatalogues, (c, r) => c.IncludeInternalCatalogues = r); - AddFlag("Cold Storage Catalogues", c => c.IncludeColdStorageCatalogues, - (c, r) => c.IncludeColdStorageCatalogues = r); AddFlag("Project Specific Catalogues", c => c.IncludeProjectSpecificCatalogues, (c, r) => c.IncludeProjectSpecificCatalogues = r); diff --git a/Rdmp.UI/PieCharts/GoodBadCataloguePieChartObjectCollection.cs b/Rdmp.UI/PieCharts/GoodBadCataloguePieChartObjectCollection.cs index b5f8b258a6..58a1fcd451 100644 --- a/Rdmp.UI/PieCharts/GoodBadCataloguePieChartObjectCollection.cs +++ b/Rdmp.UI/PieCharts/GoodBadCataloguePieChartObjectCollection.cs @@ -24,7 +24,6 @@ public class GoodBadCataloguePieChartObjectCollection : PersistableObjectCollect public bool IncludeNonExtractableCatalogues { get; set; } public bool IncludeDeprecatedCatalogues { get; set; } public bool IncludeInternalCatalogues { get; set; } - public bool IncludeColdStorageCatalogues { get; set; } public bool IncludeProjectSpecificCatalogues { get; set; } //Catalogue item filters @@ -57,9 +56,6 @@ public bool Include(Catalogue c, IDataExportRepository repo) if (status.IsProjectSpecific) returnValue &= IncludeProjectSpecificCatalogues; - if (c.IsColdStorageDataset) - returnValue &= IncludeColdStorageCatalogues; - if (c.IsDeprecated) returnValue &= IncludeDeprecatedCatalogues; @@ -100,7 +96,6 @@ public override string SaveExtraText() => { nameof(IncludeNonExtractableCatalogues), IncludeNonExtractableCatalogues.ToString() }, { nameof(IncludeDeprecatedCatalogues), IncludeDeprecatedCatalogues.ToString() }, { nameof(IncludeInternalCatalogues), IncludeInternalCatalogues.ToString() }, - { nameof(IncludeColdStorageCatalogues), IncludeColdStorageCatalogues.ToString() }, { nameof(IncludeProjectSpecificCatalogues), IncludeProjectSpecificCatalogues.ToString() }, { nameof(IncludeNonExtractableCatalogueItems), IncludeNonExtractableCatalogueItems.ToString() }, @@ -122,7 +117,6 @@ public override void LoadExtraText(string s) PersistStringHelper.GetBool(dict, nameof(IncludeNonExtractableCatalogues), true); IncludeDeprecatedCatalogues = PersistStringHelper.GetBool(dict, nameof(IncludeDeprecatedCatalogues), true); IncludeInternalCatalogues = PersistStringHelper.GetBool(dict, nameof(IncludeInternalCatalogues), true); - IncludeColdStorageCatalogues = PersistStringHelper.GetBool(dict, nameof(IncludeColdStorageCatalogues), true); IncludeProjectSpecificCatalogues = PersistStringHelper.GetBool(dict, nameof(IncludeProjectSpecificCatalogues), true); diff --git a/Rdmp.UI/SimpleDialogs/NewfindUI.cs b/Rdmp.UI/SimpleDialogs/NewfindUI.cs index 8b49fba779..2763bdda99 100644 --- a/Rdmp.UI/SimpleDialogs/NewfindUI.cs +++ b/Rdmp.UI/SimpleDialogs/NewfindUI.cs @@ -369,8 +369,6 @@ private void BuildToolStripForDatabaseObjects(RDMPCollection focusedCollection) v => UserSettings.ShowInternalCatalogues = v, "I", "Include Internal"); AddUserSettingCheckbox(() => UserSettings.ShowDeprecatedCatalogues, v => UserSettings.ShowDeprecatedCatalogues = v, "D", "Include Deprecated"); - AddUserSettingCheckbox(() => UserSettings.ShowColdStorageCatalogues, - v => UserSettings.ShowColdStorageCatalogues = v, "C", "Include Cold Storage"); AddUserSettingCheckbox(() => UserSettings.ShowProjectSpecificCatalogues, v => UserSettings.ShowProjectSpecificCatalogues = v, "P", "Include Project Specific"); AddUserSettingCheckbox(() => UserSettings.ShowNonExtractableCatalogues, diff --git a/Rdmp.UI/SimpleDialogs/Reports/MetadataReportUI.cs b/Rdmp.UI/SimpleDialogs/Reports/MetadataReportUI.cs index 331467f5cb..8cd2f2c414 100644 --- a/Rdmp.UI/SimpleDialogs/Reports/MetadataReportUI.cs +++ b/Rdmp.UI/SimpleDialogs/Reports/MetadataReportUI.cs @@ -66,7 +66,7 @@ private void btnGenerateReport_Click(object sender, EventArgs e) IEnumerable toReportOn = _catalogues; if (rbAllCatalogues.Checked) - toReportOn = toReportOn.Where(c => !c.IsInternalDataset && !c.IsColdStorageDataset && !c.IsDeprecated) + toReportOn = toReportOn.Where(c => !c.IsInternalDataset && !c.IsDeprecated) .ToArray(); else if (_cataloguesToRun == null || !_catalogues.Any()) return; diff --git a/Rdmp.UI/SimpleDialogs/SelectDialog.cs b/Rdmp.UI/SimpleDialogs/SelectDialog.cs index d508c4c946..b500e5aec5 100644 --- a/Rdmp.UI/SimpleDialogs/SelectDialog.cs +++ b/Rdmp.UI/SimpleDialogs/SelectDialog.cs @@ -448,8 +448,6 @@ private void BuildToolStripForDatabaseObjects(RDMPCollection focusedCollection) v => UserSettings.ShowInternalCatalogues = v, "I", "Include Internal"); AddUserSettingCheckbox(() => UserSettings.ShowDeprecatedCatalogues, v => UserSettings.ShowDeprecatedCatalogues = v, "D", "Include Deprecated"); - AddUserSettingCheckbox(() => UserSettings.ShowColdStorageCatalogues, - v => UserSettings.ShowColdStorageCatalogues = v, "C", "Include Cold Storage"); AddUserSettingCheckbox(() => UserSettings.ShowProjectSpecificCatalogues, v => UserSettings.ShowProjectSpecificCatalogues = v, "P", "Include Project Specific"); AddUserSettingCheckbox(() => UserSettings.ShowNonExtractableCatalogues, From 181e53511b4c9f8f876906b2ab3a3d7c646b623f Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 26 Sep 2025 07:54:56 +0100 Subject: [PATCH 063/142] re-add catalogue filter --- .../CatalogueCollectionUI.Designer.cs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs b/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs index fddb302018..639afdee97 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs @@ -29,10 +29,12 @@ private void InitializeComponent() olvFilters = new OLVColumn(); olvOrder = new OLVColumn(); imageList_RightClickIcons = new ImageList(components); + gbCatalogueFilters = new GroupBox(); catalogueCollectionFilterUI1 = new CatalogueCollectionFilterUI(); panel2 = new Panel(); tbFilter = new TextBox(); ((ISupportInitialize)tlvCatalogues).BeginInit(); + gbCatalogueFilters.SuspendLayout(); panel2.SuspendLayout(); SuspendLayout(); // @@ -88,6 +90,18 @@ private void InitializeComponent() imageList_RightClickIcons.Images.SetKeyName(3, "LOG"); imageList_RightClickIcons.Images.SetKeyName(4, "aggregates.png"); // + // gbCatalogueFilters + // + gbCatalogueFilters.Controls.Add(tbFilter); + gbCatalogueFilters.Controls.Add(catalogueCollectionFilterUI1); + gbCatalogueFilters.Dock = DockStyle.Bottom; + gbCatalogueFilters.Location = new System.Drawing.Point(0, 414); + gbCatalogueFilters.Name = "gbCatalogueFilters"; + gbCatalogueFilters.Size = new System.Drawing.Size(500, 65); + gbCatalogueFilters.TabIndex = 1; + gbCatalogueFilters.TabStop = false; + gbCatalogueFilters.Text = "Show"; + // // catalogueCollectionFilterUI1 // catalogueCollectionFilterUI1.Dock = DockStyle.Fill; @@ -99,6 +113,7 @@ private void InitializeComponent() // panel2 // panel2.Controls.Add(tlvCatalogues); + panel2.Controls.Add(gbCatalogueFilters); panel2.Dock = DockStyle.Fill; panel2.Location = new System.Drawing.Point(0, 0); panel2.Name = "panel2"; @@ -118,6 +133,8 @@ private void InitializeComponent() Name = "CatalogueCollectionUI"; Size = new System.Drawing.Size(500, 479); ((ISupportInitialize)tlvCatalogues).EndInit(); + gbCatalogueFilters.ResumeLayout(false); + gbCatalogueFilters.PerformLayout(); panel2.ResumeLayout(false); ResumeLayout(false); } @@ -128,9 +145,10 @@ private void InitializeComponent() private ImageList imageList_RightClickIcons; private OLVColumn olvColumn1; private OLVColumn olvFilters; + private GroupBox gbCatalogueFilters; private OLVColumn olvOrder; private CatalogueCollectionFilterUI catalogueCollectionFilterUI1; private Panel panel2; private TextBox tbFilter; } -} +} \ No newline at end of file From 2ee79033bfe2c5dc18cf64340dc217d6453976fd Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 26 Sep 2025 08:02:31 +0100 Subject: [PATCH 064/142] remove ui non extractable filter --- .../Providers/SearchablesMatchScorerTests.cs | 15 --------------- Rdmp.Core/Providers/SearchablesMatchScorer.cs | 12 ++++-------- .../ReusableLibraryCode/Settings/UserSettings.cs | 6 ------ .../CatalogueCollectionFilterUI.Designer.cs | 14 -------------- .../Collections/CatalogueCollectionFilterUI.cs | 7 ------- .../Filtering/CatalogueCollectionFilter.cs | 4 +--- Rdmp.UI/SimpleDialogs/NewfindUI.cs | 2 -- Rdmp.UI/SimpleDialogs/SelectDialog.cs | 2 -- 8 files changed, 5 insertions(+), 57 deletions(-) diff --git a/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs b/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs index 4ea3f7bb01..abe136451c 100644 --- a/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs +++ b/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs @@ -137,20 +137,6 @@ public void TestScoringCatalogueFlag_IsInternalDataset(bool hasFlag, bool should }, expectedResult); } - [TestCase(true, true, true)] - [TestCase(true, false, false)] - [TestCase(false, true, true)] - [TestCase(false, false, true)] - public void TestScoringCatalogueFlag_IsExtractable(bool notExtractable, bool shouldShow, bool expectedResult) - { - TestScoringFlag((c, eds) => - { - if (notExtractable) eds.DeleteInDatabase(); - - UserSettings.ShowNonExtractableCatalogues = shouldShow; - }, expectedResult); - } - [TestCase(true, true, true)] [TestCase(true, false, false)] [TestCase(false, true, true)] @@ -181,7 +167,6 @@ private void TestScoringFlag(Action setter, bool // // So set all to false to except the condition we are testing UserSettings.ShowDeprecatedCatalogues = false; - UserSettings.ShowNonExtractableCatalogues = false; UserSettings.ShowProjectSpecificCatalogues = false; UserSettings.ShowInternalCatalogues = false; diff --git a/Rdmp.Core/Providers/SearchablesMatchScorer.cs b/Rdmp.Core/Providers/SearchablesMatchScorer.cs index 33229b67ee..c0feb0794e 100644 --- a/Rdmp.Core/Providers/SearchablesMatchScorer.cs +++ b/Rdmp.Core/Providers/SearchablesMatchScorer.cs @@ -35,7 +35,6 @@ public class SearchablesMatchScorer private bool _showInternalCatalogues = true; private bool _showDeprecatedCatalogues = true; private bool _showProjectSpecificCatalogues = true; - private bool _showNonExtractableCatalogues = true; /// @@ -175,7 +174,6 @@ private void SetupRespectUserSettings() _showInternalCatalogues = !RespectUserSettings || UserSettings.ShowInternalCatalogues; _showDeprecatedCatalogues = !RespectUserSettings || UserSettings.ShowDeprecatedCatalogues; _showProjectSpecificCatalogues = !RespectUserSettings || UserSettings.ShowProjectSpecificCatalogues; - _showNonExtractableCatalogues = !RespectUserSettings || UserSettings.ShowNonExtractableCatalogues; } private int _ScoreMatches(KeyValuePair kvp, List regexes, @@ -276,7 +274,7 @@ private int _ScoreMatches(KeyValuePair private bool ScoreZeroBecauseOfUserSettings(KeyValuePair kvp) => !Filter(kvp.Key, kvp.Value, _showInternalCatalogues, _showDeprecatedCatalogues, - _showProjectSpecificCatalogues, _showNonExtractableCatalogues); + _showProjectSpecificCatalogues); private static Catalogue GetCatalogueIfAnyInDescendancy( KeyValuePair kvp) @@ -325,10 +323,9 @@ private static int MatchCount(List regexes, string str) /// /// /// - /// /// True if the item should be shown to the user based on filters public static bool Filter(object modelObject, DescendancyList descendancy, bool includeInternal, - bool includeDeprecated, bool includeProjectSpecific, bool includeNonExtractable) + bool includeDeprecated, bool includeProjectSpecific) { //doesn't relate to us... if (modelObject is not ICatalogue cata) @@ -349,12 +346,11 @@ public static bool Filter(object modelObject, DescendancyList descendancy, bool var isExtractable = cata.GetExtractabilityStatus(null) != null && cata.GetExtractabilityStatus(null).IsExtractable; - return (isExtractable && !cata.IsDeprecated && !cata.IsInternalDataset && + return (isExtractable && !cata.IsDeprecated && !cata.IsInternalDataset && !isProjectSpecific) || (includeDeprecated && cata.IsDeprecated) || (includeInternal && cata.IsInternalDataset) || - (includeProjectSpecific && isProjectSpecific) || - (includeNonExtractable && !isExtractable); + (includeProjectSpecific && isProjectSpecific); } /// diff --git a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs index 4e3f067026..82cfe307ca 100644 --- a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs +++ b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs @@ -221,12 +221,6 @@ public static bool ShowProjectSpecificCatalogues set => AppSettings.AddOrUpdateValue("ShowProjectSpecificCatalogues", value); } - public static bool ShowNonExtractableCatalogues - { - get => AppSettings.GetValueOrDefault("ShowNonExtractableCatalogues", true); - set => AppSettings.AddOrUpdateValue("ShowNonExtractableCatalogues", value); - } - /// /// True to apply theme changes to context menus and tool strips. /// diff --git a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs index 99be71cb91..7300d0879c 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs @@ -31,7 +31,6 @@ private void InitializeComponent() this.cbShowDeprecated = new System.Windows.Forms.CheckBox(); this.cbShowInternal = new System.Windows.Forms.CheckBox(); this.cbProjectSpecific = new System.Windows.Forms.CheckBox(); - this.cbShowNonExtractable = new System.Windows.Forms.CheckBox(); this.flowLayoutPanel1 = new System.Windows.Forms.FlowLayoutPanel(); this.flowLayoutPanel1.SuspendLayout(); this.SuspendLayout(); @@ -69,23 +68,11 @@ private void InitializeComponent() this.cbProjectSpecific.UseVisualStyleBackColor = true; this.cbProjectSpecific.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); // - // cbShowNonExtractable - // - this.cbShowNonExtractable.AutoSize = true; - this.cbShowNonExtractable.Location = new System.Drawing.Point(109, 26); - this.cbShowNonExtractable.Name = "cbShowNonExtractable"; - this.cbShowNonExtractable.Size = new System.Drawing.Size(102, 17); - this.cbShowNonExtractable.TabIndex = 9; - this.cbShowNonExtractable.Text = "Non Extractable"; - this.cbShowNonExtractable.UseVisualStyleBackColor = true; - this.cbShowNonExtractable.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); - // // flowLayoutPanel1 // this.flowLayoutPanel1.Controls.Add(this.cbShowInternal); this.flowLayoutPanel1.Controls.Add(this.cbShowDeprecated); this.flowLayoutPanel1.Controls.Add(this.cbProjectSpecific); - this.flowLayoutPanel1.Controls.Add(this.cbShowNonExtractable); this.flowLayoutPanel1.Dock = System.Windows.Forms.DockStyle.Fill; this.flowLayoutPanel1.Location = new System.Drawing.Point(0, 0); this.flowLayoutPanel1.Name = "flowLayoutPanel1"; @@ -108,7 +95,6 @@ private void InitializeComponent() private System.Windows.Forms.CheckBox cbShowDeprecated; private System.Windows.Forms.CheckBox cbShowInternal; private System.Windows.Forms.CheckBox cbProjectSpecific; - private System.Windows.Forms.CheckBox cbShowNonExtractable; private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel1; } } diff --git a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs index cf4591ddb8..50e7c08365 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs @@ -22,7 +22,6 @@ public CatalogueCollectionFilterUI() cbShowInternal.Checked = UserSettings.ShowInternalCatalogues; cbShowDeprecated.Checked = UserSettings.ShowDeprecatedCatalogues; cbProjectSpecific.Checked = UserSettings.ShowProjectSpecificCatalogues; - cbShowNonExtractable.Checked = UserSettings.ShowNonExtractableCatalogues; _loading = false; } @@ -37,7 +36,6 @@ private void OnCheckboxChanged(object sender, EventArgs e) UserSettings.ShowInternalCatalogues = cbShowInternal.Checked; UserSettings.ShowDeprecatedCatalogues = cbShowDeprecated.Checked; UserSettings.ShowProjectSpecificCatalogues = cbProjectSpecific.Checked; - UserSettings.ShowNonExtractableCatalogues = cbShowNonExtractable.Checked; FiltersChanged?.Invoke(this, EventArgs.Empty); } @@ -53,8 +51,6 @@ public void EnsureVisible(Catalogue c) var isExtractable = c.GetExtractabilityStatus(null); - cbShowNonExtractable.Checked = cbShowNonExtractable.Checked || isExtractable == null || - isExtractable.IsExtractable == false; } /// @@ -71,8 +67,5 @@ public void CheckForChanges() if (cbProjectSpecific.Checked != UserSettings.ShowProjectSpecificCatalogues) cbProjectSpecific.Checked = UserSettings.ShowProjectSpecificCatalogues; - - if (cbShowNonExtractable.Checked != UserSettings.ShowNonExtractableCatalogues) - cbShowNonExtractable.Checked = UserSettings.ShowNonExtractableCatalogues; } } \ No newline at end of file diff --git a/Rdmp.UI/Collections/Providers/Filtering/CatalogueCollectionFilter.cs b/Rdmp.UI/Collections/Providers/Filtering/CatalogueCollectionFilter.cs index 067949a8f9..a90c801a02 100644 --- a/Rdmp.UI/Collections/Providers/Filtering/CatalogueCollectionFilter.cs +++ b/Rdmp.UI/Collections/Providers/Filtering/CatalogueCollectionFilter.cs @@ -21,7 +21,6 @@ public class CatalogueCollectionFilter : IModelFilter private readonly bool _isInternal; private readonly bool _isDeprecated; private readonly bool _isProjectSpecific; - private readonly bool _isNonExtractable; public CatalogueCollectionFilter(ICoreChildProvider childProvider) { @@ -29,10 +28,9 @@ public CatalogueCollectionFilter(ICoreChildProvider childProvider) _isInternal = UserSettings.ShowInternalCatalogues; _isDeprecated = UserSettings.ShowDeprecatedCatalogues; _isProjectSpecific = UserSettings.ShowProjectSpecificCatalogues; - _isNonExtractable = UserSettings.ShowNonExtractableCatalogues; } public bool Filter(object modelObject) => SearchablesMatchScorer.Filter(modelObject, ChildProvider.GetDescendancyListIfAnyFor(modelObject), _isInternal, _isDeprecated, - _isProjectSpecific, _isNonExtractable); + _isProjectSpecific); } \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/NewfindUI.cs b/Rdmp.UI/SimpleDialogs/NewfindUI.cs index 2763bdda99..4d7b572fdb 100644 --- a/Rdmp.UI/SimpleDialogs/NewfindUI.cs +++ b/Rdmp.UI/SimpleDialogs/NewfindUI.cs @@ -371,8 +371,6 @@ private void BuildToolStripForDatabaseObjects(RDMPCollection focusedCollection) v => UserSettings.ShowDeprecatedCatalogues = v, "D", "Include Deprecated"); AddUserSettingCheckbox(() => UserSettings.ShowProjectSpecificCatalogues, v => UserSettings.ShowProjectSpecificCatalogues = v, "P", "Include Project Specific"); - AddUserSettingCheckbox(() => UserSettings.ShowNonExtractableCatalogues, - v => UserSettings.ShowNonExtractableCatalogues = v, "E", "Include Extractable"); } } diff --git a/Rdmp.UI/SimpleDialogs/SelectDialog.cs b/Rdmp.UI/SimpleDialogs/SelectDialog.cs index b500e5aec5..affb50df05 100644 --- a/Rdmp.UI/SimpleDialogs/SelectDialog.cs +++ b/Rdmp.UI/SimpleDialogs/SelectDialog.cs @@ -450,8 +450,6 @@ private void BuildToolStripForDatabaseObjects(RDMPCollection focusedCollection) v => UserSettings.ShowDeprecatedCatalogues = v, "D", "Include Deprecated"); AddUserSettingCheckbox(() => UserSettings.ShowProjectSpecificCatalogues, v => UserSettings.ShowProjectSpecificCatalogues = v, "P", "Include Project Specific"); - AddUserSettingCheckbox(() => UserSettings.ShowNonExtractableCatalogues, - v => UserSettings.ShowNonExtractableCatalogues = v, "E", "Include Extractable"); } } From 2a59d049db891b8b823e39f7471dc62d45735d10 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 26 Sep 2025 08:41:09 +0100 Subject: [PATCH 065/142] add make internal --- .../CommandExecution/AtomicCommandFactory.cs | 11 +++ .../ExecuteCommandMakeCatalogueInternal.cs | 77 ++++++++++++++++++ .../ExecuteCommandMakeCatalogueNotInternal.cs | 79 +++++++++++++++++++ Rdmp.Core/Providers/SearchablesMatchScorer.cs | 4 +- .../Repositories/DataExportRepository.cs | 4 +- 5 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueInternal.cs create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueNotInternal.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 177411678d..e4c1c32747 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -135,6 +135,17 @@ public IEnumerable CreateCommands(object o) SuggestedCategory = Extraction }; + yield return c.IsInternalDataset ? + new ExecuteCommandMakeCatalogueNotInternal(_activator, c) + { + Weight = -99.0008f, + SuggestedCategory = Extraction + } : new ExecuteCommandMakeCatalogueInternal(_activator, c) + { + Weight = -99.0008f, + SuggestedCategory = Extraction + }; + yield return new ExecuteCommandMakeCatalogueProjectSpecific(_activator, c, null, false) { Weight = -99.0009f, diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueInternal.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueInternal.cs new file mode 100644 index 0000000000..905bb0e768 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueInternal.cs @@ -0,0 +1,77 @@ +// Copyright (c) The University of Dundee 2018-2025 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Repositories.Construction; +using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +public class ExecuteCommandMakeCatalogueInternal : BasicCommandExecution, IAtomicCommandWithTarget +{ + private ICatalogue _catalogue; + + [UseWithObjectConstructor] + public ExecuteCommandMakeCatalogueInternal(IBasicActivateItems itemActivator, ICatalogue catalogue) : this(itemActivator) + { + SetCatalogue(catalogue); + } + + public ExecuteCommandMakeCatalogueInternal(IBasicActivateItems itemActivator) : base(itemActivator) + { + UseTripleDotSuffix = true; + } + + public override string GetCommandHelp() => + "Mark the Catalogue as Internal to restrict its extractability"; + + public override void Execute() + { + if (_catalogue == null) + SetCatalogue(SelectOne(BasicActivator.RepositoryLocator.CatalogueRepository)); + + if (_catalogue == null) + return; + + _catalogue.IsInternalDataset = true; + _catalogue.SaveToDatabase(); + Publish(_catalogue); + } + + public override Image GetImage(IIconProvider iconProvider) => BasicActivator.CoreIconProvider.GetImage(_catalogue, OverlayKind.Internal); + + public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) + { + switch (target) + { + case Catalogue catalogue: + SetCatalogue(catalogue); + break; + default: + break; + } + + return this; + } + + private void SetCatalogue(ICatalogue catalogue) + { + ResetImpossibleness(); + + _catalogue = catalogue; + + if (catalogue == null) + { + SetImpossible("Catalogue cannot be null"); + return; + } + + if (_catalogue.IsInternalDataset) + SetImpossible("Catalogue is already marked as Internal"); + } +} \ No newline at end of file diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueNotInternal.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueNotInternal.cs new file mode 100644 index 0000000000..81d27a05fc --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueNotInternal.cs @@ -0,0 +1,79 @@ +// Copyright (c) The University of Dundee 2018-2025 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.Repositories.Construction; +using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +public class ExecuteCommandMakeCatalogueNotInternal : BasicCommandExecution, IAtomicCommandWithTarget +{ + private ICatalogue _catalogue; + + [UseWithObjectConstructor] + public ExecuteCommandMakeCatalogueNotInternal(IBasicActivateItems itemActivator, ICatalogue catalogue) : this(itemActivator) + { + SetCatalogue(catalogue); + } + + public ExecuteCommandMakeCatalogueNotInternal(IBasicActivateItems itemActivator) : base(itemActivator) + { + UseTripleDotSuffix = true; + } + + public override string GetCommandHelp() => + "Mark the Catalogue as not Internal"; + + public override void Execute() + { + if (_catalogue == null) + SetCatalogue(SelectOne(BasicActivator.RepositoryLocator.CatalogueRepository)); + + if (_catalogue == null) + return; + + _catalogue.IsInternalDataset = false; + _catalogue.SaveToDatabase(); + Publish(_catalogue); + } + + public override Image GetImage(IIconProvider iconProvider) => + Image.Load(CatalogueIcons.Catalogue); + + public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) + { + switch (target) + { + case Catalogue catalogue: + SetCatalogue(catalogue); + break; + default: + break; + } + + return this; + } + + private void SetCatalogue(ICatalogue catalogue) + { + ResetImpossibleness(); + + _catalogue = catalogue; + + if (catalogue == null) + { + SetImpossible("Catalogue cannot be null"); + return; + } + + if (!_catalogue.IsInternalDataset) + SetImpossible("Catalogue is not marked as Internal"); + } +} \ No newline at end of file diff --git a/Rdmp.Core/Providers/SearchablesMatchScorer.cs b/Rdmp.Core/Providers/SearchablesMatchScorer.cs index c0feb0794e..2ee28b9ee3 100644 --- a/Rdmp.Core/Providers/SearchablesMatchScorer.cs +++ b/Rdmp.Core/Providers/SearchablesMatchScorer.cs @@ -343,10 +343,8 @@ public static bool Filter(object modelObject, DescendancyList descendancy, bool } var isProjectSpecific = cata.IsProjectSpecific(null); - var isExtractable = cata.GetExtractabilityStatus(null) != null && - cata.GetExtractabilityStatus(null).IsExtractable; - return (isExtractable && !cata.IsDeprecated && !cata.IsInternalDataset && + return (!cata.IsDeprecated && !cata.IsInternalDataset && !isProjectSpecific) || (includeDeprecated && cata.IsDeprecated) || (includeInternal && cata.IsInternalDataset) || diff --git a/Rdmp.Core/Repositories/DataExportRepository.cs b/Rdmp.Core/Repositories/DataExportRepository.cs index c2c9705f87..a3ced3893c 100644 --- a/Rdmp.Core/Repositories/DataExportRepository.cs +++ b/Rdmp.Core/Repositories/DataExportRepository.cs @@ -99,7 +99,9 @@ protected override IMapsDirectlyToDatabaseTable ConstructEntity(Type t, DbDataRe public CatalogueExtractabilityStatus GetExtractabilityStatus(ICatalogue c) { var eds = GetAllObjectsWithParent(c).ToList(); - return eds.Count == 0 ? new CatalogueExtractabilityStatus(false, false) : new CatalogueExtractabilityStatus(true, eds.Count > 1 ? true : eds.First().Projects.Any()); + var isExtractable = !c.IsInternalDataset; + var extractabilityStatus = new CatalogueExtractabilityStatus(isExtractable, eds.Count == 0 ? false : eds.First().Projects.Any()); + return extractabilityStatus; } public ISelectedDataSets[] GetSelectedDatasetsWithNoExtractionIdentifiers() => From 5776bc1fa15a5e7ea5920ef4ed6da3e912cc50d8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 26 Sep 2025 09:51:34 +0100 Subject: [PATCH 066/142] cohort selection --- .../CommandExecution/AtomicCommandFactory.cs | 26 ++++++++++++------- ...logueToCohortIdentificationSetContainer.cs | 20 +++++++++++++- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index e4c1c32747..9ae2222329 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -146,18 +146,24 @@ public IEnumerable CreateCommands(object o) SuggestedCategory = Extraction }; - yield return new ExecuteCommandMakeCatalogueProjectSpecific(_activator, c, null, false) + + if (!c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) { - Weight = -99.0009f, - SuggestedCategory = Extraction - }; - yield return new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, c, null) + yield return new ExecuteCommandMakeCatalogueProjectSpecific(_activator, c, null, false) + { + Weight = -99.0009f, + SuggestedCategory = Extraction + }; + } + else { - Weight = -99.0009f, - SuggestedCategory = Extraction, - OverrideCommandName = "Remove Project Specific Catalogue from a Project" - }; - + yield return new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, c, null) + { + Weight = -99.0009f, + SuggestedCategory = Extraction, + OverrideCommandName = "Remove Project Specific Catalogue from a Project" + }; + } yield return new ExecuteCommandSetExtractionIdentifier(_activator, c, null, null) { Weight = -99.0008f, diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs index 3e70c2c720..3a91055392 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs @@ -4,11 +4,13 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using System.Collections.Generic; using System.Linq; using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.Repositories.Construction; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; @@ -111,18 +113,34 @@ public override void Execute() // if user hasn't picked a Catalogue yet if (_catalogueCombineable == null) { + //todo only show projects that are either non-specific or in linke project + var cic = _targetCohortAggregateContainer.GetCohortIdentificationConfiguration(); + List associatedProjectCataloguesIDs= new(); + var pcica = BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(pcica => pcica.CohortIdentificationConfiguration_ID == cic.ID).FirstOrDefault(); + if(pcica is not null) + { + associatedProjectCataloguesIDs = pcica.Project.GetAllProjectCatalogues().Select(c => c.ID).ToList(); + } if (!BasicActivator.SelectObjects(new DialogArgs { WindowTitle = "Add Catalogue(s) to Container", TaskDescription = $"Choose which Catalogues to add to the cohort container '{_targetCohortAggregateContainer.Name}'. Catalogues must have at least one IsExtractionIdentifier column." - }, BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects(), out var selected)) + }, BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects().Where(c => !c.IsInternalDataset &&(!c.IsProjectSpecific(BasicActivator.RepositoryLocator.DataExportRepository) || associatedProjectCataloguesIDs.Contains(c.ID))).ToArray(), out var selected)) // user didn't pick one return; // for each catalogue they picked foreach (var catalogue in selected) { + if(BasicActivator.IsInteractive && catalogue.IsDeprecated) + { + var confirmDeprecatedUser = BasicActivator.YesNo($"{catalogue.Name} is marked as deprecated. Are you sure you wish to use it?", "Confirm use of Deprecated Catalogue"); + if (!confirmDeprecatedUser) + { + continue; + } + } var combineable = new CatalogueCombineable(catalogue); UpdateIsImpossibleFor(combineable); From fb71c1c84d71ee14804cf0a527ac26b6d0311f89 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 26 Sep 2025 10:07:11 +0100 Subject: [PATCH 067/142] limit extractions --- .../ExecuteCommandAddDatasetsToConfiguration.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs index d395395fa6..0d02dda118 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs @@ -62,7 +62,7 @@ public ExecuteCommandAddDatasetsToConfiguration(IBasicActivateItems itemActivato var _importableDataSets = childProvider.ExtractableDataSets.Except(_datasets) //where it can be used in any Project OR this project only - .Where(ds => !ds.Projects.Any()|| ds.Projects.Select(p => p.ID).Contains(targetExtractionConfiguration.Project_ID)) + .Where(ds => (!ds.Projects.Any()|| ds.Projects.Select(p => p.ID).Contains(targetExtractionConfiguration.Project_ID)) && !ds.Catalogue.IsInternalDataset) .ToArray(); SetExtractableDataSets(true, _importableDataSets); @@ -98,7 +98,10 @@ public override void Execute() return; foreach (var ds in selected) - _targetExtractionConfiguration.AddDatasetToConfiguration(ds); + if (BasicActivator.IsInteractive && ds.Catalogue.IsDeprecated && YesNo($"{ds.Catalogue.Name} is deprecated. Are you sure you wish to extract it?","Confirm use of Deprecated Catalogue")) + { + _targetExtractionConfiguration.AddDatasetToConfiguration(ds); + } } else { From 617008d9a6be5cbf3051a928c0f265bd5c7dfd6b Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 26 Sep 2025 10:28:17 +0100 Subject: [PATCH 068/142] remove extractability change --- .../WindowManagement/ActivateItems.cs | 1 - .../TestCommandsAreSupported.cs | 1 - .../CommandExecution/AtomicCommandFactory.cs | 6 -- .../ExecuteCommandChangeExtractability.cs | 88 ------------------- .../CommandExecution/BasicActivateItems.cs | 21 +++-- 5 files changed, 10 insertions(+), 107 deletions(-) delete mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs b/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs index 126afc435b..f6f05a4427 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs @@ -653,7 +653,6 @@ public override void Wait(string title, Task task, CancellationTokenSource cts) public override IEnumerable GetIgnoredCommands() { yield return typeof(ExecuteCommandRefreshObject); - yield return typeof(ExecuteCommandChangeExtractability); yield return typeof(ExecuteCommandOpenInExplorer); yield return typeof(ExecuteCommandDeletePlugin); yield return typeof(ExecuteCommandCreateNewFileBasedProcessTask); diff --git a/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs b/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs index d57a3c5c19..67ae4e50c1 100644 --- a/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs +++ b/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs @@ -67,7 +67,6 @@ public void Init() [TestCase(typeof(ExecuteCommandAssociateCatalogueWithLoadMetadata))] [TestCase(typeof(ExecuteCommandAssociateCohortIdentificationConfigurationWithProject))] [TestCase(typeof(ExecuteCommandBulkImportTableInfos))] - [TestCase(typeof(ExecuteCommandChangeExtractability))] [TestCase(typeof(ExecuteCommandChangeExtractionCategory))] [TestCase(typeof(ExecuteCommandChangeLoadStage))] [TestCase(typeof(ExecuteCommandCheck))] diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 9ae2222329..844890ebcd 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -129,12 +129,6 @@ public IEnumerable CreateCommands(object o) }; if (!isApiCall) { - yield return new ExecuteCommandChangeExtractability(_activator, c) - { - Weight = -99.0010f, - SuggestedCategory = Extraction - }; - yield return c.IsInternalDataset ? new ExecuteCommandMakeCatalogueNotInternal(_activator, c) { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs deleted file mode 100644 index 24593de385..0000000000 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) The University of Dundee 2018-2019 -// This file is part of the Research Data Management Platform (RDMP). -// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -// You should have received a copy of the GNU General Public License along with RDMP. If not, see . - -using System.Linq; -using Rdmp.Core.Curation.Data; -using Rdmp.Core.DataExport.Data; -using Rdmp.Core.Icons.IconProvision; -using Rdmp.Core.Providers; -using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; -using SixLabors.ImageSharp; -using SixLabors.ImageSharp.PixelFormats; - -namespace Rdmp.Core.CommandExecution.AtomicCommands; - -public sealed class ExecuteCommandChangeExtractability : BasicCommandExecution -{ - private readonly Catalogue _catalogue; - private readonly bool _markExtractable; - - public ExecuteCommandChangeExtractability(IBasicActivateItems activator, Catalogue catalogue, - bool? explicitExtractability = null) : base(activator) - { - _catalogue = catalogue; - var status = _catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository); - if (status == null) - { - SetImpossible( - "We don't know whether Catalogue is extractable or not (possibly no Data Export database is available)"); - return; - } - - if (status.IsProjectSpecific) - { - SetImpossible( - "Cannot change the extractability because it is configured as a 'Project Specific Catalogue'"); - return; - } - - // mark it extractable true/false as passed in constructor or just flip its state - _markExtractable = explicitExtractability ?? !status.IsExtractable; - } - - public override string GetCommandName() => _markExtractable ? "Mark Extractable" : "Mark Not Extractable"; - - public override string GetCommandHelp() => - !_markExtractable - ? "Prevent dataset from being released in Project extracts. This fails if it is already part of any ExtractionConfigurations" - : @"Enable dataset linkage\extraction in Project extracts. This requires that at least one column be marked IsExtractionIdentifier"; - - public override Image GetImage(IIconProvider iconProvider) => - iconProvider.GetImage(RDMPConcept.ExtractableDataSet, _markExtractable ? OverlayKind.Add : OverlayKind.Delete); - - public override void Execute() - { - base.Execute(); - - if (_markExtractable) - { - if (_catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository).IsExtractable) - { - Show($"{_catalogue} is already extractable"); - } - else - { - new ExtractableDataSet(BasicActivator.RepositoryLocator.DataExportRepository, _catalogue); - Publish(_catalogue); - } - } - else - { - var extractabilityRecord = - ((DataExportChildProvider)BasicActivator.CoreChildProvider).ExtractableDataSets.SingleOrDefault(ds => - ds.Catalogue_ID == _catalogue.ID); - if (extractabilityRecord == null) - { - Show($"{_catalogue} is already non-extractable"); - } - else - { - extractabilityRecord.DeleteInDatabase(); - Publish(_catalogue); - } - } - } -} \ No newline at end of file diff --git a/Rdmp.Core/CommandExecution/BasicActivateItems.cs b/Rdmp.Core/CommandExecution/BasicActivateItems.cs index 5f1b8cf779..84cf904b72 100644 --- a/Rdmp.Core/CommandExecution/BasicActivateItems.cs +++ b/Rdmp.Core/CommandExecution/BasicActivateItems.cs @@ -380,21 +380,20 @@ protected virtual bool InteractiveDelete(IDeleteable deletable) { case Catalogue c: { - if (c.GetExtractabilityStatus(RepositoryLocator.DataExportRepository).IsExtractable) + var extractableDataSets = RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", c.ID); + if (extractableDataSets.Any()) { - if (YesNo( - "Catalogue must first be made non extractable before it can be deleted, mark non extractable?", - "Make Non Extractable")) + foreach (var ds in extractableDataSets) { - var cmd = new ExecuteCommandChangeExtractability(this, c); - cmd.Execute(); - } - else - { - return false; + var selectedDatasets = RepositoryLocator.DataExportRepository.GetAllObjectsWhere("ExtractableDataSet_ID", ds.ID); + if (selectedDatasets.Any()) + { + this.Show("Catalogue is used in a number of projects. Remove this catalogue from all projects to allow deletion"); + return false; + } } + foreach (var ds in extractableDataSets) ds.DeleteInDatabase(); } - break; } case ExtractionFilter f: From 77b6aeb5838ac26d65dcc15887f11a0954d3c906 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 08:34:28 +0100 Subject: [PATCH 069/142] add try catch --- Rdmp.Core/Curation/SimpleStringValueEncryption.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs index cf7cdf7748..5da92bf79f 100644 --- a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs +++ b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs @@ -36,7 +36,14 @@ public class SimpleStringValueEncryption : IEncryptStrings public SimpleStringValueEncryption(string parameters) { - _turing.FromXmlString(parameters ?? Key); + Console.WriteLine(parameters ?? Key); + try + { + _turing.FromXmlString(parameters ?? Key); + }catch(Exception e) + { + Console.WriteLine(e.Message); + } } /// From c8013d84bd9d4a5fa7a2846c4f9105794151d7ef Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 08:50:05 +0100 Subject: [PATCH 070/142] bump runner --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c213a232b1..f99de72237 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: setup .NET - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 - name: Populate Databases.yaml shell: bash run: | @@ -86,7 +86,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: setup .NET - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 - name: Populate Databases.yaml shell: bash run: | @@ -154,13 +154,13 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: setup .NET - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 - name: Determine RDMP build version id: version shell: bash run: perl -ne "print \"rdmpversion=\$1\n\" if /AssemblyInformationalVersion\(\"([0-9a-z.-]+)\"\)/;" SharedAssemblyInfo.cs >> $GITHUB_OUTPUT - name: Setup .NET Core - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 with: dotnet-version: 7.0.x - name: BundleSource @@ -293,7 +293,7 @@ jobs: path: ${{ github.workspace }}/ key: ${{ github.sha }}-your-cache-key-bundled - name: setup .NET - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 - name: Determine RDMP build version id: version shell: bash From 1731aa687ecd6441b969bc7a8abce084f8082219 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 09:35:53 +0100 Subject: [PATCH 071/142] add key --- .github/workflows/build.yml | 6 ++++++ Rdmp.Core/Curation/SimpleStringValueEncryption.cs | 14 ++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f99de72237..742e87e4fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,6 +49,12 @@ jobs: run: dotnet build --configuration Release --verbosity minimal - name: Initialise RDMP run: | + $rsa = New-Object System.Security.Cryptography.RSACryptoServiceProvider -ArgumentList 2048 + $SavePath=".\keys" + New-Item $SavePath -ItemType directory | Out-Null + $rsa.toXmlString($true) | Out-File $SavePath\private-key.xml + $rsa.ToXmlString($false) | Out-File $SavePath\public-key.xml + $RDMP_KEY_LOCATION=$SavePath\private-key.xml dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 --collation "Latin1_General_CS_AI" "(localdb)\MSSQLLocalDB" TEST_ -e - name: Create MySql Logging, DQE and Cohort Building Cache Db run: | diff --git a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs index 5da92bf79f..780ed694f8 100644 --- a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs +++ b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs @@ -36,14 +36,11 @@ public class SimpleStringValueEncryption : IEncryptStrings public SimpleStringValueEncryption(string parameters) { - Console.WriteLine(parameters ?? Key); - try - { - _turing.FromXmlString(parameters ?? Key); - }catch(Exception e) - { - Console.WriteLine(e.Message); - } + _turing.FromXmlString(parameters ?? Key); + var x = _turing.ToXmlString(true); + var y = _turing.ToXmlString(true); + Console.WriteLine("2"); + } /// @@ -53,6 +50,7 @@ public SimpleStringValueEncryption(string parameters) /// public string Encrypt(string toEncrypt) { + // Fall back on bad encryption if no private key is configured if (_turing.KeySize < 1024) return string.Join('-', From 45b9ee3f4b02d145aaa07c6c0ccdac897ec7f1de Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 09:43:12 +0100 Subject: [PATCH 072/142] add quotes --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 742e87e4fb..51e7a7eb89 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,7 +54,7 @@ jobs: New-Item $SavePath -ItemType directory | Out-Null $rsa.toXmlString($true) | Out-File $SavePath\private-key.xml $rsa.ToXmlString($false) | Out-File $SavePath\public-key.xml - $RDMP_KEY_LOCATION=$SavePath\private-key.xml + $RDMP_KEY_LOCATION="$SavePath\private-key.xml" dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 --collation "Latin1_General_CS_AI" "(localdb)\MSSQLLocalDB" TEST_ -e - name: Create MySql Logging, DQE and Cohort Building Cache Db run: | From 9026dbf5306c2899d027f69b48b43476db09dd74 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 09:52:13 +0100 Subject: [PATCH 073/142] tidy up --- Rdmp.Core/Curation/SimpleStringValueEncryption.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs index 780ed694f8..46ec6565d3 100644 --- a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs +++ b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs @@ -36,10 +36,8 @@ public class SimpleStringValueEncryption : IEncryptStrings public SimpleStringValueEncryption(string parameters) { + Console.WriteLine(parameters ?? Key); _turing.FromXmlString(parameters ?? Key); - var x = _turing.ToXmlString(true); - var y = _turing.ToXmlString(true); - Console.WriteLine("2"); } From e8b8a272e5bc24b45124f35d4eec062f73b7fff3 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 09:59:40 +0100 Subject: [PATCH 074/142] update key --- .../Curation/SimpleStringValueEncryption.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs index 46ec6565d3..b2d35370f2 100644 --- a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs +++ b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs @@ -24,14 +24,14 @@ public class SimpleStringValueEncryption : IEncryptStrings private const string Key = @" - AQAB - sMDeszVErUmbqOxQavw5OsWpL3frccEGtTJYM8G54Fw7NK6xFVUrq79nWB6px4/B -

6kcXnTVJrVuD9j6qUm+F71jIL2H92lgN

- wSRbrdj1qGBPBnYMO5dx11gvfNCKKdWF - aKdxaQzQ6Nwkyu+bbk/baNwkMOZ5W/xR - B/B8rErM3l0HIpbbrd9t2JJRcWoJI+sZ - NFv4Z26nbMpOkOcAnO3rktoMffza+3Ul - Y8zC8dUF7gI9zeeAkKfReInauV6wpg4iVh7jaTDN5DAmKFURTAyv6Il6LEyr07JB + 5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ== + AQAB +

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

+6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8= +ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U= +KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck= +BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ= +q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ== "; public SimpleStringValueEncryption(string parameters) From 431b42faea92afc1bed34cf94a9a579052463930 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 10:10:29 +0100 Subject: [PATCH 075/142] add env --- .github/workflows/build.yml | 6 +++--- .../Curation/SimpleStringValueEncryption.cs | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 51e7a7eb89..58b610ab8a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,12 +49,12 @@ jobs: run: dotnet build --configuration Release --verbosity minimal - name: Initialise RDMP run: | - $rsa = New-Object System.Security.Cryptography.RSACryptoServiceProvider -ArgumentList 2048 + $text=@"5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" $SavePath=".\keys" New-Item $SavePath -ItemType directory | Out-Null - $rsa.toXmlString($true) | Out-File $SavePath\private-key.xml - $rsa.ToXmlString($false) | Out-File $SavePath\public-key.xml + $text| Out-File $SavePath\private-key.xml $RDMP_KEY_LOCATION="$SavePath\private-key.xml" + $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 --collation "Latin1_General_CS_AI" "(localdb)\MSSQLLocalDB" TEST_ -e - name: Create MySql Logging, DQE and Cohort Building Cache Db run: | diff --git a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs index b2d35370f2..4a460f5ecc 100644 --- a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs +++ b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs @@ -22,16 +22,16 @@ public class SimpleStringValueEncryption : IEncryptStrings private readonly RSACryptoServiceProvider _turing = new(); private const string Key = - @" + @" - 5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ== - AQAB -

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

-6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8= -ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U= -KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck= -BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ= -q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ== + AQAB + sMDeszVErUmbqOxQavw5OsWpL3frccEGtTJYM8G54Fw7NK6xFVUrq79nWB6px4/B +

6kcXnTVJrVuD9j6qUm+F71jIL2H92lgN

+ wSRbrdj1qGBPBnYMO5dx11gvfNCKKdWF + aKdxaQzQ6Nwkyu+bbk/baNwkMOZ5W/xR + B/B8rErM3l0HIpbbrd9t2JJRcWoJI+sZ + NFv4Z26nbMpOkOcAnO3rktoMffza+3Ul + Y8zC8dUF7gI9zeeAkKfReInauV6wpg4iVh7jaTDN5DAmKFURTAyv6Il6LEyr07JB "; public SimpleStringValueEncryption(string parameters) From 29093abc608ee5fc92ed31530f16193604f48184 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 10:15:44 +0100 Subject: [PATCH 076/142] remove here string --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 58b610ab8a..922b3b0dda 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,7 +49,7 @@ jobs: run: dotnet build --configuration Release --verbosity minimal - name: Initialise RDMP run: | - $text=@"5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" + $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" $SavePath=".\keys" New-Item $SavePath -ItemType directory | Out-Null $text| Out-File $SavePath\private-key.xml From 5938de08a4acac682853ff0b2cd8d8fa323d8a64 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 10:22:10 +0100 Subject: [PATCH 077/142] tidy up build --- .github/workflows/build.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 922b3b0dda..386bfdd55d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,14 @@ jobs: CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; EOF + - name: Set RSA Key + run: | + $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" + $SavePath=".\keys" + New-Item $SavePath -ItemType directory | Out-Null + $text| Out-File $SavePath\private-key.xml + $RDMP_KEY_LOCATION="$SavePath\private-key.xml" + $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" - name: Build run: | dotnet build --configuration Release --verbosity minimal @@ -49,12 +57,6 @@ jobs: run: dotnet build --configuration Release --verbosity minimal - name: Initialise RDMP run: | - $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" - $SavePath=".\keys" - New-Item $SavePath -ItemType directory | Out-Null - $text| Out-File $SavePath\private-key.xml - $RDMP_KEY_LOCATION="$SavePath\private-key.xml" - $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 --collation "Latin1_General_CS_AI" "(localdb)\MSSQLLocalDB" TEST_ -e - name: Create MySql Logging, DQE and Cohort Building Cache Db run: | @@ -100,6 +102,14 @@ jobs: CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; EOF + - name: Set RSA Key + run: | + $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" + $SavePath=".\keys" + New-Item $SavePath -ItemType directory | Out-Null + $text| Out-File $SavePath\private-key.xml + $RDMP_KEY_LOCATION="$SavePath\private-key.xml" + $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" - name: Build run: | dotnet build --configuration Release --verbosity minimal From 0d2754467ff02716eb813903669eabd7ba486f0b Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 10:25:25 +0100 Subject: [PATCH 078/142] more env setting --- .github/workflows/build.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 386bfdd55d..00088c782f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,14 +57,23 @@ jobs: run: dotnet build --configuration Release --verbosity minimal - name: Initialise RDMP run: | + $SavePath=".\keys" + $RDMP_KEY_LOCATION="$SavePath\private-key.xml" + $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 --collation "Latin1_General_CS_AI" "(localdb)\MSSQLLocalDB" TEST_ -e - name: Create MySql Logging, DQE and Cohort Building Cache Db run: | + $SavePath=".\keys" + $RDMP_KEY_LOCATION="$SavePath\private-key.xml" + $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/ dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/ dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/ - name: Run integration test scripts run: | + $SavePath=".\keys" + $RDMP_KEY_LOCATION="$SavePath\private-key.xml" + $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_list_destroy_catalogue.yaml dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_cohort.yaml dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_dataload.yaml From ce441fab2a4408ec5a7d400eafcc2b5b227e24ce Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 10:33:15 +0100 Subject: [PATCH 079/142] add for tests --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 00088c782f..646cbc3ba9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,6 +81,9 @@ jobs: - name: Test (DB) shell: bash run: | + $SavePath=".\keys" + $RDMP_KEY_LOCATION="$SavePath\private-key.xml" + $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" rm -rf coverage curl -L "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" > minio.exe ./minio.exe server ./minio --console-address :9001 & From c63a66dba320a4affa7b641844b4d92941bd4a5d Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 10:44:43 +0100 Subject: [PATCH 080/142] update key location --- .github/workflows/build.yml | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 646cbc3ba9..ddeb5d2b8e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,8 @@ jobs: tests_db: name: Run Database Tests runs-on: windows-latest + env: + RDMP_KEY_LOCATION: C:\temp\key\private-key.xml steps: - name: Checkout code uses: actions/checkout@v4 @@ -29,12 +31,10 @@ jobs: EOF - name: Set RSA Key run: | + $SavePath=" C:\temp\key" + New-Item -ItemType Directory -Force -Path $SavePath | Out-Null $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" - $SavePath=".\keys" - New-Item $SavePath -ItemType directory | Out-Null $text| Out-File $SavePath\private-key.xml - $RDMP_KEY_LOCATION="$SavePath\private-key.xml" - $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" - name: Build run: | dotnet build --configuration Release --verbosity minimal @@ -57,23 +57,14 @@ jobs: run: dotnet build --configuration Release --verbosity minimal - name: Initialise RDMP run: | - $SavePath=".\keys" - $RDMP_KEY_LOCATION="$SavePath\private-key.xml" - $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 --collation "Latin1_General_CS_AI" "(localdb)\MSSQLLocalDB" TEST_ -e - name: Create MySql Logging, DQE and Cohort Building Cache Db run: | - $SavePath=".\keys" - $RDMP_KEY_LOCATION="$SavePath\private-key.xml" - $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/ dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/ dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/ - name: Run integration test scripts run: | - $SavePath=".\keys" - $RDMP_KEY_LOCATION="$SavePath\private-key.xml" - $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_list_destroy_catalogue.yaml dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_cohort.yaml dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_dataload.yaml @@ -81,9 +72,6 @@ jobs: - name: Test (DB) shell: bash run: | - $SavePath=".\keys" - $RDMP_KEY_LOCATION="$SavePath\private-key.xml" - $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" rm -rf coverage curl -L "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" > minio.exe ./minio.exe server ./minio --console-address :9001 & From 5c5d05444f62b45481234454d29e053e4d040be0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 10:49:00 +0100 Subject: [PATCH 081/142] use correct drive --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ddeb5d2b8e..371e35b9cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: name: Run Database Tests runs-on: windows-latest env: - RDMP_KEY_LOCATION: C:\temp\key\private-key.xml + RDMP_KEY_LOCATION: D:\a\_temp\key\private-key.xml steps: - name: Checkout code uses: actions/checkout@v4 @@ -31,7 +31,7 @@ jobs: EOF - name: Set RSA Key run: | - $SavePath=" C:\temp\key" + $SavePath=" D:\a\_temp\key" New-Item -ItemType Directory -Force -Path $SavePath | Out-Null $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" $text| Out-File $SavePath\private-key.xml From 0b062caabbabe98a99efc41f9bf829ca97c25dbc Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 10:52:44 +0100 Subject: [PATCH 082/142] use temp --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 371e35b9cc..725151f8cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: name: Run Database Tests runs-on: windows-latest env: - RDMP_KEY_LOCATION: D:\a\_temp\key\private-key.xml + RDMP_KEY_LOCATION: $env:TEMP\key\private-key.xml steps: - name: Checkout code uses: actions/checkout@v4 @@ -31,7 +31,7 @@ jobs: EOF - name: Set RSA Key run: | - $SavePath=" D:\a\_temp\key" + $SavePath="$env:TEMP\key" New-Item -ItemType Directory -Force -Path $SavePath | Out-Null $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" $text| Out-File $SavePath\private-key.xml From 7ae0d78a0d309f830a89e6b61ed615f45d0b00ee Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 10:58:47 +0100 Subject: [PATCH 083/142] use temp --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 725151f8cc..81b7f56ff2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: name: Run Database Tests runs-on: windows-latest env: - RDMP_KEY_LOCATION: $env:TEMP\key\private-key.xml + RDMP_KEY_LOCATION: D:\a\RDMP\RDMP\key\private-key.xml steps: - name: Checkout code uses: actions/checkout@v4 @@ -31,7 +31,7 @@ jobs: EOF - name: Set RSA Key run: | - $SavePath="$env:TEMP\key" + $SavePath="D:\a\RDMP\RDMP\key" New-Item -ItemType Directory -Force -Path $SavePath | Out-Null $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" $text| Out-File $SavePath\private-key.xml From d626f4e1d5fa14eaef04f526493fadec42bbd4a3 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 12:03:23 +0100 Subject: [PATCH 084/142] updaye build --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 81b7f56ff2..73b946d365 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,6 +90,8 @@ jobs: tests_file_system: name: Run File System Tests runs-on: windows-latest + env: + RDMP_KEY_LOCATION: D:\a\RDMP\RDMP\key\private-key.xml steps: - name: Checkout code uses: actions/checkout@v4 @@ -104,12 +106,10 @@ jobs: EOF - name: Set RSA Key run: | + $SavePath="D:\a\RDMP\RDMP\key" + New-Item -ItemType Directory -Force -Path $SavePath | Out-Null $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" - $SavePath=".\keys" - New-Item $SavePath -ItemType directory | Out-Null $text| Out-File $SavePath\private-key.xml - $RDMP_KEY_LOCATION="$SavePath\private-key.xml" - $env:RDMP_KEY_LOCATION="$SavePath\private-key.xml" - name: Build run: | dotnet build --configuration Release --verbosity minimal From 8c6d6c238dc2640abb7a2d549981ec7144c3e103 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Sep 2025 12:58:30 +0100 Subject: [PATCH 085/142] tidy up --- ...cuteCommandAddCatalogueToCohortIdentificationSetContainer.cs | 1 - Rdmp.Core/Curation/SimpleStringValueEncryption.cs | 2 -- 2 files changed, 3 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs index 3a91055392..3adaf4efbd 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs @@ -113,7 +113,6 @@ public override void Execute() // if user hasn't picked a Catalogue yet if (_catalogueCombineable == null) { - //todo only show projects that are either non-specific or in linke project var cic = _targetCohortAggregateContainer.GetCohortIdentificationConfiguration(); List associatedProjectCataloguesIDs= new(); var pcica = BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(pcica => pcica.CohortIdentificationConfiguration_ID == cic.ID).FirstOrDefault(); diff --git a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs index 4a460f5ecc..a35ee07d46 100644 --- a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs +++ b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs @@ -36,9 +36,7 @@ public class SimpleStringValueEncryption : IEncryptStrings public SimpleStringValueEncryption(string parameters) { - Console.WriteLine(parameters ?? Key); _turing.FromXmlString(parameters ?? Key); - } /// From c0af3f981c9ef2cba15e0b2d34e765edadd096a0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 2 Oct 2025 14:37:19 +0100 Subject: [PATCH 086/142] add changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98c7fbcd2e..12420bef14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add Data Load component to allow SQL to be executed on an external database server - Improve Extraction Log Viewer Filter - Add Dataset Variable document to extractions +- Update extractability to only allow non-internal projects to be extracted +- Remove functionality to mark individual catalogues as extractable/not extractable. Favor use of marking as internal. ## [9.0.1] - 2025-07-31 From 24e993c0bcd6df93ce1545147a5eb5d75cfc7f60 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 2 Oct 2025 14:41:28 +0100 Subject: [PATCH 087/142] update from codeql --- .../ExecuteCommandAddDatasetsToConfiguration.cs | 11 +++++------ Rdmp.Core/Providers/SearchablesMatchScorer.cs | 10 +++++----- Rdmp.Core/Repositories/DataExportRepository.cs | 3 +-- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs index 0d02dda118..37319f9457 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs @@ -62,7 +62,7 @@ public ExecuteCommandAddDatasetsToConfiguration(IBasicActivateItems itemActivato var _importableDataSets = childProvider.ExtractableDataSets.Except(_datasets) //where it can be used in any Project OR this project only - .Where(ds => (!ds.Projects.Any()|| ds.Projects.Select(p => p.ID).Contains(targetExtractionConfiguration.Project_ID)) && !ds.Catalogue.IsInternalDataset) + .Where(ds => (!ds.Projects.Any() || ds.Projects.Select(p => p.ID).Contains(targetExtractionConfiguration.Project_ID)) && !ds.Catalogue.IsInternalDataset) .ToArray(); SetExtractableDataSets(true, _importableDataSets); @@ -97,11 +97,10 @@ public override void Execute() }, _toadd.Cast().ToArray(), out var selected)) return; - foreach (var ds in selected) - if (BasicActivator.IsInteractive && ds.Catalogue.IsDeprecated && YesNo($"{ds.Catalogue.Name} is deprecated. Are you sure you wish to extract it?","Confirm use of Deprecated Catalogue")) - { - _targetExtractionConfiguration.AddDatasetToConfiguration(ds); - } + foreach (var ds in selected.Where(ds => BasicActivator.IsInteractive && ds.Catalogue.IsDeprecated && YesNo($"{ds.Catalogue.Name} is deprecated. Are you sure you wish to extract it?", "Confirm use of Deprecated Catalogue"))) + { + _targetExtractionConfiguration.AddDatasetToConfiguration(ds); + } } else { diff --git a/Rdmp.Core/Providers/SearchablesMatchScorer.cs b/Rdmp.Core/Providers/SearchablesMatchScorer.cs index 2ee28b9ee3..4f603bbf55 100644 --- a/Rdmp.Core/Providers/SearchablesMatchScorer.cs +++ b/Rdmp.Core/Providers/SearchablesMatchScorer.cs @@ -344,11 +344,11 @@ public static bool Filter(object modelObject, DescendancyList descendancy, bool var isProjectSpecific = cata.IsProjectSpecific(null); - return (!cata.IsDeprecated && !cata.IsInternalDataset && - !isProjectSpecific) || - (includeDeprecated && cata.IsDeprecated) || - (includeInternal && cata.IsInternalDataset) || - (includeProjectSpecific && isProjectSpecific); + var isVisible = !cata.IsDeprecated && !cata.IsInternalDataset && !isProjectSpecific; + var showDeprecated = includeDeprecated && cata.IsDeprecated; + var showInternal = includeInternal && cata.IsInternalDataset; + var showProjectSpecific = includeProjectSpecific && isProjectSpecific; + return isVisible || showDeprecated || showInternal || showProjectSpecific; } /// diff --git a/Rdmp.Core/Repositories/DataExportRepository.cs b/Rdmp.Core/Repositories/DataExportRepository.cs index a3ced3893c..98970d6939 100644 --- a/Rdmp.Core/Repositories/DataExportRepository.cs +++ b/Rdmp.Core/Repositories/DataExportRepository.cs @@ -100,8 +100,7 @@ public CatalogueExtractabilityStatus GetExtractabilityStatus(ICatalogue c) { var eds = GetAllObjectsWithParent(c).ToList(); var isExtractable = !c.IsInternalDataset; - var extractabilityStatus = new CatalogueExtractabilityStatus(isExtractable, eds.Count == 0 ? false : eds.First().Projects.Any()); - return extractabilityStatus; + var extractabilityStatus = new CatalogueExtractabilityStatus(isExtractable, eds.Count != 0 && eds.First().Projects.Any()); } public ISelectedDataSets[] GetSelectedDatasetsWithNoExtractionIdentifiers() => From f9e424b67e322baa78e17ce2bda0ae7123a51ac9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 2 Oct 2025 14:42:05 +0100 Subject: [PATCH 088/142] fix build --- Rdmp.Core/Repositories/DataExportRepository.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Rdmp.Core/Repositories/DataExportRepository.cs b/Rdmp.Core/Repositories/DataExportRepository.cs index 98970d6939..4ae7b264c3 100644 --- a/Rdmp.Core/Repositories/DataExportRepository.cs +++ b/Rdmp.Core/Repositories/DataExportRepository.cs @@ -101,6 +101,7 @@ public CatalogueExtractabilityStatus GetExtractabilityStatus(ICatalogue c) var eds = GetAllObjectsWithParent(c).ToList(); var isExtractable = !c.IsInternalDataset; var extractabilityStatus = new CatalogueExtractabilityStatus(isExtractable, eds.Count != 0 && eds.First().Projects.Any()); + return extractabilityStatus; } public ISelectedDataSets[] GetSelectedDatasetsWithNoExtractionIdentifiers() => From 121c340dbf090e303d6e43082ed732221ce6e74c Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 2 Oct 2025 15:54:53 +0100 Subject: [PATCH 089/142] fix project specific --- .../CommandExecution/AtomicCommandFactory.cs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 844890ebcd..f1c1bf3861 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -141,15 +141,14 @@ public IEnumerable CreateCommands(object o) }; - if (!c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) + + yield return new ExecuteCommandMakeCatalogueProjectSpecific(_activator, c, null, false) { - yield return new ExecuteCommandMakeCatalogueProjectSpecific(_activator, c, null, false) - { - Weight = -99.0009f, - SuggestedCategory = Extraction - }; - } - else + Weight = -99.0009f, + SuggestedCategory = Extraction + }; + + if (c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) { yield return new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, c, null) { From f883190c238daa4d61f29cb9c46353a835d255f6 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 13 Oct 2025 08:20:55 +0100 Subject: [PATCH 090/142] Task/rdmp 333 export catalogues to confluence (#2238) * add basic write to confluence * with links * working updates * tidy up code * add smmary docs * refactor confluence builder * fix refactor * tidy up --- CHANGELOG.md | 1 + ...cuteCommandExportCataloguesToConfluence.cs | 279 ++++++++++++++++++ .../Confluence/ConfluencePageBuilder.cs | 234 +++++++++++++++ 3 files changed, 514 insertions(+) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs create mode 100644 Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 12420bef14..81449f5e66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add Dataset Variable document to extractions - Update extractability to only allow non-internal projects to be extracted - Remove functionality to mark individual catalogues as extractable/not extractable. Favor use of marking as internal. +- Add cli command to export catalogue metadata to a Atlassian Confluence space ## [9.0.1] - 2025-07-31 diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs new file mode 100644 index 0000000000..17f16fdae6 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs @@ -0,0 +1,279 @@ +using Newtonsoft.Json; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Dataset.Confluence; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands +{ + /// + /// Uses provided Confluence credentials to populate a tree of pages about the public catalogues in RDMP + /// + public class ExecuteCommandExportCataloguesToConfluence : BasicCommandExecution, IAtomicCommand + { + + private readonly IBasicActivateItems _activator; + private readonly string _subdomain; + private readonly int _spaceId; + private readonly string _apiKey; + private readonly string _owner; + private readonly string _description; + private readonly HttpClient _client = new(); + private readonly Dictionary _cataloguePageLookups = []; + private string rootParentId = null; + private int rootParentVersion = 0; + private class ConfluencePageBody + { + public string representation { get; set; } = "storage"; + public string value { get; set; } + } + + private class ConfluencePageVersion + { + public int number { get; set; } + } + + private class ConfluencePagePostRequest + { + public string spaceId { get; set; } + public string status { get; set; } = "current"; + public string title { get; set; } = "Catalogues"; + public string parentId { get; set; } + public ConfluencePageBody body { get; set; } + + } + + private class ConfluencePagePutRequest + { + public int id { get; set; } + public string status { get; set; } = "current"; + public string title { get; set; } = "Catalogues"; + public ConfluencePageBody body { get; set; } + public ConfluencePageVersion version { get; set; } + + } + + private class ConfluenceLinks + { + public string webui { get; set; } + } + + private class ConfluencePostError + { + public int status { get; set; } + public string code { get; set; } + public string title { get; set; } + public string detail { get; set; } + } + private class ConfluencePostErrors + { + public List errors { get; set; } + } + + private class ConfluenceGetResults + { + public List results { get; set; } + } + private class ConfluenceGetResult + { + public ConfluencePageVersion version { get; set; } + public string id { get; set; } + } + + private class ConfluencePageResponse + { + public string id { get; set; } + public ConfluenceLinks _links { get; set; } + public ConfluencePageVersion version { get; set; } + } + + public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, string subdomain, int spaceId, string apiKey, string owner, string description) + { + _activator = activator; + _subdomain = subdomain; + _spaceId = spaceId; + _apiKey = apiKey; + _owner = owner; + _description = description; + } + + private static ConfluencePageResponse ResponseToConfluenceResponseObject(HttpResponseMessage response) + { + var content = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + return JsonConvert.DeserializeObject(content); + } + + private static ConfluencePostErrors PostResponseToConfluenceErrorsObject(HttpResponseMessage response) + { + var content = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + return JsonConvert.DeserializeObject(content); + } + + private ConfluenceGetResults GetConfluencePage(string uri, string title) + { + var result = Task.Run(async () => await _client.GetAsync($"{uri}?title={title}&space-id={_spaceId}")).Result; + var content = Task.Run(async () => await result.Content.ReadAsStringAsync()).Result; + return JsonConvert.DeserializeObject(content); + } + + + private void UpdateContainerPage(ConfluencePageBuilder builder, string uri) + { + var rootPageHTML = builder.BuildContainerPage(_cataloguePageLookups); + var putRequest = new ConfluencePagePutRequest() + { + id = int.Parse(rootParentId), + title = $"{_owner} Catalogues", + status = "current", + body = new ConfluencePageBody() { value = rootPageHTML }, + version = new ConfluencePageVersion() + { + number = rootParentVersion + 1 + } + }; + var response = Task.Run(async () => await _client.PutAsJsonAsync($"{uri}/{rootParentId}", putRequest)).Result; + if (response.StatusCode == System.Net.HttpStatusCode.OK) + { + var putResponseContent = ResponseToConfluenceResponseObject(response); + rootParentId = putResponseContent.id; + rootParentVersion = putResponseContent.version.number; + } + else + { + throw new Exception("Unable to update container page"); + } + } + + private bool PageAlreadyExists(ConfluencePostErrors errors) + { + return errors.errors.Count == 1 && errors.errors[0].title.Contains("page with this title already exists"); + } + + private void CreateContainerPage(ConfluencePageBuilder builder, string uri) + { + var rootPageHTML = builder.BuildContainerPage(_cataloguePageLookups); + var request = new ConfluencePagePostRequest() + { + spaceId = _spaceId.ToString(), + title = $"{_owner} Catalogues", + body = new ConfluencePageBody() { value = rootPageHTML } + }; + HttpResponseMessage response = Task.Run(async () => await _client.PostAsJsonAsync(uri, request)).Result; + if (response.StatusCode == System.Net.HttpStatusCode.OK) + { + var confluencePostResponseObject = ResponseToConfluenceResponseObject(response); + rootParentId = confluencePostResponseObject.id; + rootParentVersion = confluencePostResponseObject.version.number; + } + else if (response.StatusCode == System.Net.HttpStatusCode.BadRequest) + { + var confluenceErrorsResponse = PostResponseToConfluenceErrorsObject(response); + if (PageAlreadyExists(confluenceErrorsResponse)) + { + var foundPages = GetConfluencePage(uri, request.title); + if (foundPages.results.Count != 1) + { + throw new Exception($"Multiple pages in pace {_spaceId} with name {request.title}"); + } + rootParentId = foundPages.results[0].id; + rootParentVersion = foundPages.results[0].version.number; + UpdateContainerPage(builder, uri); + } + else + { + throw new Exception($"Unexpected Error when creating container page - {confluenceErrorsResponse.errors[0].title}"); + } + } + else + { + throw new Exception($"Unable to create root page - {response.StatusCode}"); + } + } + + private void CreateCataloguePage(Catalogue catalogue, ConfluencePageBuilder builder, string uri) + { + var cataloguePageHTML = builder.BuildHTMLForCatalogue(catalogue); + var request = new ConfluencePagePostRequest() + { + spaceId = _spaceId.ToString(), + title = catalogue.Name, + parentId = rootParentId, + body = new ConfluencePageBody() { value = cataloguePageHTML } + }; + var response = Task.Run(async () => await _client.PostAsJsonAsync(uri, request)).Result; + if (response.StatusCode == System.Net.HttpStatusCode.OK) + { + var responseContent = ResponseToConfluenceResponseObject(response); + _cataloguePageLookups.TryAdd(catalogue.ID, responseContent._links.webui); + } + else if (response.StatusCode == System.Net.HttpStatusCode.BadRequest) + { + var responseContent = PostResponseToConfluenceErrorsObject(response); + if (PageAlreadyExists(responseContent)) + { + var getResults = GetConfluencePage(uri, request.title); + if (getResults.results.Count != 1) + { + throw new Exception($"Multiple pages in pace {_spaceId} with name {request.title}"); + } + var id = getResults.results[0].id; + var version = getResults.results[0].version.number; + + var cataloguePutRequest = new ConfluencePagePutRequest() + { + id = int.Parse(id), + title = request.title, + status = "current", + body = new ConfluencePageBody() { value = cataloguePageHTML }, + version = new ConfluencePageVersion() + { + number = version + 1 + } + }; + response = Task.Run(async () => await _client.PutAsJsonAsync($"{uri}/{id}", cataloguePutRequest)).Result; + if (response.StatusCode != System.Net.HttpStatusCode.OK) + { + throw new Exception($"Unable to PUT update for catalogue '{catalogue.Name}' - {response.StatusCode}"); + } + } + } + else + { + throw new Exception($"Unknown status code when working with catalogue '{catalogue.Name}' - {response.StatusCode}"); + } + } + + public override void Execute() + { + base.Execute(); + _client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", _apiKey); + + var catalogues = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects() + .Where(c => !c.IsDeprecated && !c.IsInternalDataset && !c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) + .ToList(); + var builder = new ConfluencePageBuilder(catalogues, _owner, _description, _subdomain); + var uri = $"https://{_subdomain}.atlassian.net/wiki/api/v2/pages"; + + CreateContainerPage(builder, uri); + if (rootParentId != null) + { + foreach (var catalogue in catalogues) + { + CreateCataloguePage(catalogue, builder, uri); + } + UpdateContainerPage(builder, uri);//re-add the links for the catalogue pages to the home page + + } + else + { + throw new Exception($"Unable to find root id in response"); + + } + } + + } +} diff --git a/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs b/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs new file mode 100644 index 0000000000..517ee3b70b --- /dev/null +++ b/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs @@ -0,0 +1,234 @@ +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Rdmp.Core.Dataset.Confluence +{ + /// + /// Used to generate HTML to create a confluence tree of + /// - Overview Page + /// - Each catalogue + /// Tabulates dataset variables and metadata about each catalogue + /// + public class ConfluencePageBuilder + { + + private readonly List _catalogues; + + private readonly string _repositoryName = ""; + private readonly string _repositoryDescription = ""; + private readonly string _subdomain = ""; + public ConfluencePageBuilder(List catalogues, string name,string description, string subdomain) { + _catalogues = catalogues; + _repositoryName = name; + _repositoryDescription = description; + _subdomain = subdomain; + } + + private string BuildCatalogueOverviewHTML(Catalogue catalogue,string pageId) + { + string name = pageId != null ?$"{catalogue.Name}" : catalogue.Name; + + return $""" +

{catalogue.Name}

+ + + + + + + + + + + +
+ Dataset Name + + Description + + Tags +
+ {name} + + {catalogue.Description} + + {string.Join(", ",catalogue.Search_keywords)} +
+ """; + } + + public string BuildContainerPage(Dictionary cataloguePageLookup) { + return $""" + +

{_repositoryName} Catalogues

+

{_repositoryDescription}

+

+ {string.Join("",_catalogues.Select(c => { cataloguePageLookup.TryGetValue(c.ID, out var pageId); return BuildCatalogueOverviewHTML(c, pageId); }))} + """; + } + + private string SplitCamelCase(string input) + { + return System.Text.RegularExpressions.Regex.Replace(input.Replace(",", ", "), "([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim(); + + } + + private static string BuildDataVariableRecord(CatalogueItem catalogueItem) + { + var lookups = catalogueItem.CatalogueRepository.GetAllObjectsWhere("ForeignKey_ID", catalogueItem.ColumnInfo.ID); + return $""" + + {catalogueItem.Name} + {catalogueItem.ColumnInfo.Data_type} + {catalogueItem.ExtractionInformation.IsPrimaryKey} + {catalogueItem.Description} + {catalogueItem.ExtractionInformation.IsExtractionIdentifier} + {lookups.Length != 0} + + """; + } + + public string BuildHTMLForCatalogue(Catalogue catalogue) + { + //todo think about splitting and spacing of some of these values + return $""" +

Dataset Summary

+

{catalogue.Description}

+

+

Data Details

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Resource Type{SplitCamelCase(Enum.GetName(catalogue.Type))}
Dataset Purpose{SplitCamelCase(Enum.GetName(catalogue.Purpose))}
Dataset Type{SplitCamelCase(catalogue.DataType)}
Dataset SubType{SplitCamelCase(catalogue.DataSubType)}
Dataset Source{SplitCamelCase(catalogue.DataSource)}
Dataset Source Setting{SplitCamelCase(catalogue.DataSourceSetting)}
Dataset Purpose{SplitCamelCase(Enum.GetName(catalogue.Purpose))}
Dataset Keywords{catalogue.Search_keywords}
+

Geospatial and Temporal Details

+ + + + + + + + + + + + + + + + + +
Geographic Coverage{catalogue.Geographical_coverage}
Granularity{catalogue.Granularity}
Start Date{catalogue.StartDate}
End Date{catalogue.EndDate}
+

Access Details

+ + + + + + + + + + + + + + + + + +
Access Contact{catalogue.Contact_details}
Data Controller{catalogue.DataController}
Data Processor{catalogue.DataProcessor}
Jurisdiction{catalogue.Juristiction}
+

Attribution

+ + + + + + + + + + + + + + + + + +
DOI{catalogue.Doi}
Controlled Vocabulary{catalogue.ControlledVocabulary}
Associated People{catalogue.AssociatedPeople}
Jurisdiction{catalogue.Juristiction}
+

Data Updates

+ + + + + + + + + + + + + +
Update Frequency{Enum.GetName(catalogue.Update_freq)}
Initial Release Date{catalogue.DatasetReleaseDate}
Update Lag{Enum.GetName(catalogue.UpdateLag)}
+

Dataset Variables

+ + + + + + + + + + {string.Join("",catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation is not null).Select(ci => BuildDataVariableRecord(ci)))} +
+ Variable Name + + Type + + Null Possible (Y/N) + + Description + + Identifier + + Has Lookups +
+ """; + } + } +} From c784a9058e955c2078cec2eb8d081e5f029f4131 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 13 Oct 2025 11:23:02 +0100 Subject: [PATCH 091/142] Move RSA Loading to runtime (#2240) * move checks * add changelog --- CHANGELOG.md | 1 + Rdmp.Core/Curation/SimpleStringValueEncryption.cs | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81449f5e66..a193d10bd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add Dataset Variable document to extractions - Update extractability to only allow non-internal projects to be extracted - Remove functionality to mark individual catalogues as extractable/not extractable. Favor use of marking as internal. +- Move RSA key checks to fix bug where bad RSA keys were causing issues at launch - Add cli command to export catalogue metadata to a Atlassian Confluence space diff --git a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs index a35ee07d46..674378bfdb 100644 --- a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs +++ b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs @@ -34,9 +34,11 @@ public class SimpleStringValueEncryption : IEncryptStrings Y8zC8dUF7gI9zeeAkKfReInauV6wpg4iVh7jaTDN5DAmKFURTAyv6Il6LEyr07JB "; + private readonly string _parameters; + public SimpleStringValueEncryption(string parameters) { - _turing.FromXmlString(parameters ?? Key); + _parameters=parameters; } /// @@ -46,7 +48,7 @@ public SimpleStringValueEncryption(string parameters) /// public string Encrypt(string toEncrypt) { - + _turing.FromXmlString(_parameters ?? Key); // Fall back on bad encryption if no private key is configured if (_turing.KeySize < 1024) return string.Join('-', @@ -71,6 +73,7 @@ public string Encrypt(string toEncrypt) /// public string Decrypt(string toDecrypt) { + _turing.FromXmlString(_parameters ?? Key); if (toDecrypt.StartsWith("$js1$", StringComparison.Ordinal) && toDecrypt.EndsWith("$", StringComparison.Ordinal)) { // Good, it's a new-style AES+RSA encrypted string From 2fa95845bb6fb1d6bb487b4feacafa63ba25e660 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 21 Oct 2025 11:41:48 +0100 Subject: [PATCH 092/142] Task/rdmp 339 prep v9.0.3 (#2242) * fix extraction add catalogues * remove filter * update rsa * revert key * update deps * update release date * delete key * add try * remove rsa key from gh actions * fix runner * update tests * use test env variable * add check * remove test * tidy up * change RSA key * re-add tests * tidy up * fix test * tidy up * remove password test for new key * update deps * tidy up --- .github/workflows/build.yml | 18 +--------- CHANGELOG.md | 3 +- Directory.Packages.props | 36 +++++++++---------- .../Curation/Integration/EncryptionTests.cs | 17 --------- .../PasswordEncryptionKeyLocationTests.cs | 3 -- ...xecuteCommandAddDatasetsToConfiguration.cs | 2 +- .../ExecuteCommandCreateNewFilter.cs | 11 ------ .../Curation/SimpleStringValueEncryption.cs | 20 +++++------ 8 files changed, 31 insertions(+), 79 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 73b946d365..726b61e76b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,8 +15,6 @@ jobs: tests_db: name: Run Database Tests runs-on: windows-latest - env: - RDMP_KEY_LOCATION: D:\a\RDMP\RDMP\key\private-key.xml steps: - name: Checkout code uses: actions/checkout@v4 @@ -29,12 +27,6 @@ jobs: CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; EOF - - name: Set RSA Key - run: | - $SavePath="D:\a\RDMP\RDMP\key" - New-Item -ItemType Directory -Force -Path $SavePath | Out-Null - $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" - $text| Out-File $SavePath\private-key.xml - name: Build run: | dotnet build --configuration Release --verbosity minimal @@ -90,8 +82,6 @@ jobs: tests_file_system: name: Run File System Tests runs-on: windows-latest - env: - RDMP_KEY_LOCATION: D:\a\RDMP\RDMP\key\private-key.xml steps: - name: Checkout code uses: actions/checkout@v4 @@ -104,12 +94,6 @@ jobs: CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; EOF - - name: Set RSA Key - run: | - $SavePath="D:\a\RDMP\RDMP\key" - New-Item -ItemType Directory -Force -Path $SavePath | Out-Null - $text="5CBYA/4GKu57eepMM5dHguZR6QxPakujvUPq81YQlAs9XdAS5OugT9xYATXV0ZUVeQrtCOj1jjS6cSnekJKzXMD48H2IbT+ImRVqyjE19dgeeZtK1cGa8wDKTdNtjo+ur2iaMzItE3VChcMidWncQpiieieSUwQ81uoab7foVdaQm078TzlHLaWiSyAPCOOIeeO2q7HEjVPkbiqCGl2Lkrzvzct84SDMFkyQXzXVJCfdlFn5bX3/8OwC9gWVICPBVbVFZZQ3skUKFqK/aYcgJL/svDyhFsj89TK3xzz8YE1r8VwxVtvqLfRXrWqUCV1n2vEm4XUjuTwQi2nwclREuQ==AQAB

+zfBbR8e3gLc5TDOSoyjclKdPyl62BZvp0kdlmihI5rJ/Bk+CnymYr22vBbWe/wJugWL4bLEAMWiWsa0ri0mJig24aJZ+DMEJDh+wz4J70OBbsE7jydw9whkt6r/8dYCwE2L4aKlL5pUOL/DzgQJ6vxkQ3UAjYMvEzBPapd7Z7c=

6HgO3YVz2umROVrHkQ0xa+4a+EMZVAEBhS+ZuJ5KhXBomxYptfAud0WmGN1zOM7TpBYnk8IBx2kuBKoAavFtjbjsoINVxlLlUvUDEJmeElVz5TqRRCNEChY0sfDlR9gVIopB/p7BU5SFRz5i7+qpsWzWdBU/BR93K2vJNsj2Vw8=ecv4bY1vC7hbnIrjGWXCQMUpE9xqgKWwEGz0eV3U8kwzrZQXbkIs8SaFl/+Cka4KkTPrM8vWF4G6S0SXiPK+0jUhFpf+AsXJNj5lxwcnDeeusyHgXHGE5WAeZKX1XSyjPNTcAtM2PzQVrUXcCuAOZu1jNwlc8T8u7aC4gDddT1U=KNlP42Ub4o/AUQ++maJz2L9SReWkgbpbhgfDP0mxVplWCEpwseOuho7ajOv83zKYxfCOq8wfe+bjizZENIaP9aNVES+C1wKiAV3EWBpmSFpzrwgHlq2LuyoDwHDQGTvDGvqodhF3bzRd5xLzV60ofGDfni5NkJzi1+JszQ+rGck=BC4M2rtw/lHKW8gDVcQSB1a1yWlgtLqtoX+krelqO59/6Np2ApsPc43SUoy4PY1f+Oxf+Erik1NM1+TRucVBGB8AP1q0SFuTsmWiLE9zv/1zjeJJLOoPbpGia+bQ/r7fP+ZhBK8ldae7FAOctcoSfQ0jAn2IBpDyvlAlcnwRvwQ=q1e/w//gEg7dn0xjv7w4chEcJLaiT2xQp6+DoRFbklZ+2R+XkWmJF3KghwgweSJI5olWUALprM3d23FfQaduIJSwZbFj7upxZsm3U/ZyWRzihuQk6ThpcWt+h8Xt283/nrAqYZmmUZ8ZP+64ywef8EVEhAuE0+Wy7JkZEiBH2W/MEXUvbMV8w282/X6H8zpIkHgjMvy/rouDMFA+ZLR9OOCofw7aVV9VivOVCVIhWe+inrQzG3UCLEEmKNOy0FmqQYvZ4vtwJ+kAByo6xW2YO9cHtEJFiKrZ1O2A0P0xtziOqStDq6JqoeE/bty8y3oM3HPyXMZXG2ecLuwbP4usoQ==" - $text| Out-File $SavePath\private-key.xml - name: Build run: | dotnet build --configuration Release --verbosity minimal @@ -323,4 +307,4 @@ jobs: file: dist/* tag: ${{ github.ref }} overwrite: true - file_glob: true + file_glob: true \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index a193d10bd0..88a3705c27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [9.0.2] - Unreleased +## [9.0.2] - 2025-10-21 - Add ability to view logs as a flat list. Default settings can be updated via user settings -- allow the use of Catalogue CIC filters in coresponding Extractions - Add Data Load component to allow SQL to be executed on an external database server - Improve Extraction Log Viewer Filter - Add Dataset Variable document to extractions diff --git a/Directory.Packages.props b/Directory.Packages.props index d932ac78ce..a257a13832 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,30 +1,30 @@ - - - - + + + + - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - + + + + - - + + @@ -33,17 +33,17 @@ - - + + - + - - + + - + \ No newline at end of file diff --git a/Rdmp.Core.Tests/Curation/Integration/EncryptionTests.cs b/Rdmp.Core.Tests/Curation/Integration/EncryptionTests.cs index 9c8b30abfc..336e9fbc97 100644 --- a/Rdmp.Core.Tests/Curation/Integration/EncryptionTests.cs +++ b/Rdmp.Core.Tests/Curation/Integration/EncryptionTests.cs @@ -240,21 +240,4 @@ public void MigrationOfOldPasswordsTest() creds.DeleteInDatabase(); } } - - [Test] - public void PasswordTooLong() - { - if (RepositoryLocator.CatalogueRepository.EncryptionManager is PasswordEncryptionKeyLocation em && - !string.IsNullOrWhiteSpace(em.GetKeyFileLocation())) - Assert.Inconclusive( - "Could not run test because there is already an encryption key set up. Likely one that handles very long passwords"); - - var password = "a"; - for (var i = 0; i < 200; i++) - password += "a"; - - var ex = Assert.Throws(() => TestFreakyPasswordValues(password)); - Assert.That( - ex.Message, Is.EqualTo("The free text Value supplied to this class was too long to be encrypted (Length of string was 201)")); - } } \ No newline at end of file diff --git a/Rdmp.Core.Tests/Curation/Integration/PasswordEncryptionKeyLocationTests.cs b/Rdmp.Core.Tests/Curation/Integration/PasswordEncryptionKeyLocationTests.cs index ddfa9cec62..4502437df9 100644 --- a/Rdmp.Core.Tests/Curation/Integration/PasswordEncryptionKeyLocationTests.cs +++ b/Rdmp.Core.Tests/Curation/Integration/PasswordEncryptionKeyLocationTests.cs @@ -31,7 +31,6 @@ protected override void SetUp() public void NoKeyFileToStartWith() { var keyLocation = new PasswordEncryptionKeyLocation(CatalogueRepository); - //there shouldn't already be a key Assert.That(keyLocation.GetKeyFileLocation(), Is.Null); @@ -79,7 +78,6 @@ public void Encrypt() Console.WriteLine($"Encrypted (stock) is:{encrypter.Value}"); Console.WriteLine($"Decrypted (stock) is:{encrypter.GetDecryptedValue()}"); - var keyLocation = new PasswordEncryptionKeyLocation(CatalogueRepository); keyLocation.CreateNewKeyFile(Path.Combine(TestContext.CurrentContext.TestDirectory, "my.key")); var p = keyLocation.OpenKeyFile(); @@ -88,7 +86,6 @@ public void Encrypt() var s = CatalogueRepository.EncryptionManager.GetEncrypter(); var exception = Assert.Throws(() => s.Decrypt(encrypter.Value)); - Assert.That(exception.Message, Does.StartWith("Could not decrypt an encrypted string, possibly you are trying to decrypt it after having changed the PrivateKey ")); var encrypted = s.Encrypt(value); Console.WriteLine($"Encrypted (with key) is:{encrypted}"); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs index 37319f9457..9b6b8ace9e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs @@ -97,7 +97,7 @@ public override void Execute() }, _toadd.Cast().ToArray(), out var selected)) return; - foreach (var ds in selected.Where(ds => BasicActivator.IsInteractive && ds.Catalogue.IsDeprecated && YesNo($"{ds.Catalogue.Name} is deprecated. Are you sure you wish to extract it?", "Confirm use of Deprecated Catalogue"))) + foreach (var ds in selected.Where(ds => !ds.Catalogue.IsDeprecated || (BasicActivator.IsInteractive && ds.Catalogue.IsDeprecated && YesNo($"{ds.Catalogue.Name} is deprecated. Are you sure you wish to extract it?", "Confirm use of Deprecated Catalogue")))) { _targetExtractionConfiguration.AddDatasetToConfiguration(ds); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs index 3755d0e694..a4b01172e9 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs @@ -205,17 +205,6 @@ public override void Execute() var c = GetCatalogue(); _offerFilters = c?.GetAllFilters(); - if (_host is SelectedDataSets sds) - { - var cohortId = sds.ExtractionConfiguration.Cohort.OriginID; - var cic = c.CatalogueRepository.GetObjectByID(cohortId); - if (cic != null) - { - var filters = cic.RootCohortAggregateContainer.GetAllAggregateConfigurationsRecursively().SelectMany(ac => ac.RootFilterContainer.GetAllFiltersIncludingInSubContainersRecursively()); - filters = filters.Where(f => f.GetCatalogue().ID == c.ID); - _offerFilters = _offerFilters.Concat(filters).ToArray(); - } - } if (_offerFilters == null || !_offerFilters.Any()) SetImpossible($"There are no Filters declared in Catalogue '{c?.ToString() ?? "NULL"}'"); diff --git a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs index 674378bfdb..439fc505d2 100644 --- a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs +++ b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs @@ -23,16 +23,16 @@ public class SimpleStringValueEncryption : IEncryptStrings private const string Key = @" - - AQAB - sMDeszVErUmbqOxQavw5OsWpL3frccEGtTJYM8G54Fw7NK6xFVUrq79nWB6px4/B -

6kcXnTVJrVuD9j6qUm+F71jIL2H92lgN

- wSRbrdj1qGBPBnYMO5dx11gvfNCKKdWF - aKdxaQzQ6Nwkyu+bbk/baNwkMOZ5W/xR - B/B8rErM3l0HIpbbrd9t2JJRcWoJI+sZ - NFv4Z26nbMpOkOcAnO3rktoMffza+3Ul - Y8zC8dUF7gI9zeeAkKfReInauV6wpg4iVh7jaTDN5DAmKFURTAyv6Il6LEyr07JB -"; + + xZ7V6twqlj+3L1hsjB+BwhzQEgcs8tpqzgSjwgjPwzfisLFBN9HkwX2u+ZrgrKhKmZgSOVBYRhZiGJeF1hY5gLz3/Zo05fcvtgL0ylEhi6wnH9y6CPu8+4xCOJ26eywtNS02v2CC/0HUWnw6kNHBqdyyNjwuC12Ll5YJ3wZ1kQZ2mce3DMjpgRDvYqnm8ldwLWpgreK7I5vggQgons3v39x+Cx501Eo0qd+iHzXf6/8pUhYqb+xhDdb9gnHcitipXVFG3Ts27OUiuO0uLAO4Es400ApyWpvHdCCNrBJ/EQWZGEO/qVKmfP0CWBmokVL+IUawX9lPl/Luo9W5AFpaIQ== + AQAB +

xvQflEOfLYAXiMFrG/alxcgUbimbY8Qc1/3Jm21CFVrkQL1rNQ/PEXavXYyNP3jRwUrqKbJrtcdz5MPgKpvA/iWdqjp4Qx+V7N2DkdzKL7liViKoCHf1eXfyeWkCErHOgInTerOSlCaeABqkqpr+eaAtzy2j4df5U/1+j8g7I48=

+ /kjapfiD7aC1yQX8BYiJ29oTlvZx0sptfXA0Bx9qXvd1UFTqjDbVoIF7tXk+DsottRL97G52ImpYf4w4wJFb1F+d5sQCaCrEzIdQ/4mJdpn7FLYxErYXpoZl0e2knIoqQie7+vaw2oV9YSx3iJHoEVfDM+0NAW+aCUUJGNorD08= + mAYms0ZQtZXxZcBWNhHsbgsLAXqtkDhkye7VRPzhyCuhyo5zAyLHWVLVgahKrjuGHCtAbwg1Ibv8pMu/2Q8XE5xus4rmJnRWPZ6uUKDjpkAEEkl9GKuBWYX8NCW3Pc28O6AVhub8lFRF21KAjRTOauWo22zGk2ZS0IkdUoTwG6U= + nQsrllNMT0bw3k0G3/f6hEBD1vkvROrmAhF44GlDjZEw78Lx9FStTOqLF4HglMvCvNEU55808H5TV7qnFi7v0tKWt32YqvK3BkYP/THZJtlkWt9GoXK6WosoeSVWg6NFBAR8MTuH7/1/eLM4w6yw8X0NPpWJcbiWHmF3g9TBwTs= + fbI4APSSbOOd07goBWzZYjdrEVEZDiboK4jFKYoMI19PrbQBTvpRmLsJxhg96fgjiFfmw05tsPIsLSQPczsav6JDq25sQyOaz7VE33Y8CEV4rVvI+l7vlWqvn/EfThPqYdR4UOHa6G5wcBTI48O4+SmKMhLQl1GK/ZA+XEmlHx8= + QC+Uv1F/K4nKT8BikShylr+Q/SoDeWVjp0Juhcki4f82y7jmu+CachYGTN/29V07zaNM1/y2jx0aA27Dc4OIbb3ythXt9HtSrcVMCKJNSPZDRuAENIK/INyvbYAdX4A7trfWvlX0dj/FXxZWV08pnagm4eKt+dcKTdPXpO6OJOnnSYMcdRBFdZLAj/sh8oMNXMHSxsCA7YHjL+4NGbpyTMYnfa6CUxFc36cLi2/Hm6JhV1nYEswTI5T1qRqmWpF1+H0ksxLKx6I5D61mzfaf7RQPJRCLhddXoLPIh8IaoNIomPcZyZQBxwBhtgj8pf9rcBN50aqNc5V2vKoEOt/gGQ== + "; private readonly string _parameters; From 5b63c3c7954821c2d928f594d835c0df77265899 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 21 Oct 2025 15:31:29 +0100 Subject: [PATCH 093/142] Allow RSA keys to be accessed via network shares (#2245) * update key * tidy up --- CHANGELOG.md | 1 + .../Managers/PasswordEncryptionKeyLocation.cs | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a3705c27..99aa6b8516 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Remove functionality to mark individual catalogues as extractable/not extractable. Favor use of marking as internal. - Move RSA key checks to fix bug where bad RSA keys were causing issues at launch - Add cli command to export catalogue metadata to a Atlassian Confluence space +- Allow the use of netowrk share RSA keys ## [9.0.1] - 2025-07-31 diff --git a/Rdmp.Core/Repositories/Managers/PasswordEncryptionKeyLocation.cs b/Rdmp.Core/Repositories/Managers/PasswordEncryptionKeyLocation.cs index fc151723b5..7fbb20189e 100644 --- a/Rdmp.Core/Repositories/Managers/PasswordEncryptionKeyLocation.cs +++ b/Rdmp.Core/Repositories/Managers/PasswordEncryptionKeyLocation.cs @@ -65,7 +65,7 @@ private string GetKeyFileLocationImpl() public string OpenKeyFile() { var location = GetKeyFileLocation(); - return location is null ? null : File.ReadAllText(location); + return location is null ? null : File.ReadAllText($@"{location}"); } private static void DeserializeFromLocation(string keyLocation) @@ -74,7 +74,7 @@ private static void DeserializeFromLocation(string keyLocation) return; try { - new RSACryptoServiceProvider().FromXmlString(File.ReadAllText(keyLocation)); + new RSACryptoServiceProvider().FromXmlString(File.ReadAllText($@"{keyLocation}")); } catch (Exception ex) { @@ -129,8 +129,8 @@ public FileInfo CreateNewKeyFile(string path) public void ChangeLocation(string newLocation) { ClearAllInjections(); - - if (!File.Exists(newLocation)) + + if (!File.Exists($@"{newLocation}")) throw new FileNotFoundException($"Could not find key file at:{newLocation}"); //confirms that it is accessible and deserializable From e363c41251e8609847320f5c142fa01eea1bf002 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 24 Oct 2025 08:10:35 +0100 Subject: [PATCH 094/142] add safer default pipeline --- .../ExecuteCommandAddDatasetsToConfiguration.cs | 15 +++++++++++++-- .../ExecuteCommandExportCataloguesToConfluence.cs | 4 ++-- .../DataExport/Data/ExtractionConfiguration.cs | 11 +++++------ SharedAssemblyInfo.cs | 6 +++--- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs index 9b6b8ace9e..25880568e5 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs @@ -4,6 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using System; using System.Linq; using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.DataExport.Data; @@ -62,7 +63,17 @@ public ExecuteCommandAddDatasetsToConfiguration(IBasicActivateItems itemActivato var _importableDataSets = childProvider.ExtractableDataSets.Except(_datasets) //where it can be used in any Project OR this project only - .Where(ds => (!ds.Projects.Any() || ds.Projects.Select(p => p.ID).Contains(targetExtractionConfiguration.Project_ID)) && !ds.Catalogue.IsInternalDataset) + .Where(ds => + { + try + { + return (!ds.Projects.Any() || ds.Projects.Select(p => p.ID).Contains(targetExtractionConfiguration.Project_ID)) && !ds.Catalogue.IsInternalDataset; + } + catch (Exception) + { + return false; + } + }) .ToArray(); SetExtractableDataSets(true, _importableDataSets); @@ -97,7 +108,7 @@ public override void Execute() }, _toadd.Cast().ToArray(), out var selected)) return; - foreach (var ds in selected.Where(ds => !ds.Catalogue.IsDeprecated || (BasicActivator.IsInteractive && ds.Catalogue.IsDeprecated && YesNo($"{ds.Catalogue.Name} is deprecated. Are you sure you wish to extract it?", "Confirm use of Deprecated Catalogue")))) + foreach (var ds in selected.Where(ds => !ds.Catalogue.IsDeprecated || (BasicActivator.IsInteractive && ds.Catalogue.IsDeprecated && YesNo($"{ds.Catalogue.Name} is deprecated. Are you sure you wish to extract it?", "Confirm use of Deprecated Catalogue")))) { _targetExtractionConfiguration.AddDatasetToConfiguration(ds); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs index 17f16fdae6..a9069bfba3 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs @@ -18,7 +18,7 @@ public class ExecuteCommandExportCataloguesToConfluence : BasicCommandExecution, private readonly IBasicActivateItems _activator; private readonly string _subdomain; - private readonly int _spaceId; + private readonly string _spaceId; private readonly string _apiKey; private readonly string _owner; private readonly string _description; @@ -91,7 +91,7 @@ private class ConfluencePageResponse public ConfluencePageVersion version { get; set; } } - public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, string subdomain, int spaceId, string apiKey, string owner, string description) + public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, string subdomain, string spaceId, string apiKey, string owner, string description) { _activator = activator; _subdomain = subdomain; diff --git a/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs b/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs index 0c164559ff..f40425d45a 100644 --- a/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs +++ b/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs @@ -220,12 +220,11 @@ public IReleaseLog[] ReleaseLog /// [NoMappingToDatabase] - public IPipeline DefaultPipeline => - DefaultPipeline_ID == null - ? null - : (IPipeline)((IDataExportRepository)Repository).CatalogueRepository.GetObjectByID( - DefaultPipeline_ID.Value); - + //public IPipeline DefaultPipeline => + // DefaultPipeline_ID == null + // ? null + // : ((IDataExportRepository)Repository).CatalogueRepository.GetAllObjects().FirstOrDefault(p => p.ID == DefaultPipeline_ID); + public IPipeline DefaultPipeline => DefaultPipeline_ID == null?null: ((IDataExportRepository) Repository).CatalogueRepository.GetAllObjects().FirstOrDefault(p => p.ID == DefaultPipeline_ID); /// [NoMappingToDatabase] diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index e9664cf2b3..f1d5a9ccb4 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("9.0.2")] -[assembly: AssemblyFileVersion("9.0.2")] -[assembly: AssemblyInformationalVersion("9.0.2")] +[assembly: AssemblyVersion("9.0.3")] +[assembly: AssemblyFileVersion("9.0.3")] +[assembly: AssemblyInformationalVersion("9.0.3-prerelease-for-sindy")] From 4cae2db0798f6c155207aae21c9e376999fe5279 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 24 Oct 2025 09:37:22 +0100 Subject: [PATCH 095/142] add changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99aa6b8516..0f82713b61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.0.3] - Unreleased +- Improve checking for default pipelines + ## [9.0.2] - 2025-10-21 - Add ability to view logs as a flat list. Default settings can be updated via user settings - Add Data Load component to allow SQL to be executed on an external database server From 14882b4f6e73e5856f72b5e0593b13d41b3c4cf8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 24 Oct 2025 11:55:01 +0100 Subject: [PATCH 096/142] add changerlog --- CHANGELOG.md | 3 ++- Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs | 1 + SharedAssemblyInfo.cs | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f82713b61..26cf775609 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.0.3] - Unreleased - Improve checking for default pipelines +- Improve confluence catalogue metadata extraction ## [9.0.2] - 2025-10-21 - Add ability to view logs as a flat list. Default settings can be updated via user settings @@ -16,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Remove functionality to mark individual catalogues as extractable/not extractable. Favor use of marking as internal. - Move RSA key checks to fix bug where bad RSA keys were causing issues at launch - Add cli command to export catalogue metadata to a Atlassian Confluence space -- Allow the use of netowrk share RSA keys +- Allow the use of network share RSA keys ## [9.0.1] - 2025-07-31 diff --git a/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs b/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs index 517ee3b70b..4e0b4aba3e 100644 --- a/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs +++ b/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs @@ -71,6 +71,7 @@ public string BuildContainerPage(Dictionary cataloguePageLookup) { private string SplitCamelCase(string input) { + if (input is null) return input; return System.Text.RegularExpressions.Regex.Replace(input.Replace(",", ", "), "([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim(); } diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index f1d5a9ccb4..7f6033d29f 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: AssemblyVersion("9.0.3")] [assembly: AssemblyFileVersion("9.0.3")] -[assembly: AssemblyInformationalVersion("9.0.3-prerelease-for-sindy")] +[assembly: AssemblyInformationalVersion("9.0.3")] From 229ecfdaa69fabdb73240f83d043f75418db3f10 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 24 Oct 2025 13:02:21 +0100 Subject: [PATCH 097/142] tidy up --- Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs b/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs index f40425d45a..ce7aa73506 100644 --- a/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs +++ b/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs @@ -220,10 +220,6 @@ public IReleaseLog[] ReleaseLog /// [NoMappingToDatabase] - //public IPipeline DefaultPipeline => - // DefaultPipeline_ID == null - // ? null - // : ((IDataExportRepository)Repository).CatalogueRepository.GetAllObjects().FirstOrDefault(p => p.ID == DefaultPipeline_ID); public IPipeline DefaultPipeline => DefaultPipeline_ID == null?null: ((IDataExportRepository) Repository).CatalogueRepository.GetAllObjects().FirstOrDefault(p => p.ID == DefaultPipeline_ID); /// From 6120e97a2b35b53ae3ed830e11ddd69b748ba7d2 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 24 Oct 2025 13:15:18 +0100 Subject: [PATCH 098/142] revert confluence --- .../ExecuteCommandExportCataloguesToConfluence.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs index a9069bfba3..17f16fdae6 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs @@ -18,7 +18,7 @@ public class ExecuteCommandExportCataloguesToConfluence : BasicCommandExecution, private readonly IBasicActivateItems _activator; private readonly string _subdomain; - private readonly string _spaceId; + private readonly int _spaceId; private readonly string _apiKey; private readonly string _owner; private readonly string _description; @@ -91,7 +91,7 @@ private class ConfluencePageResponse public ConfluencePageVersion version { get; set; } } - public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, string subdomain, string spaceId, string apiKey, string owner, string description) + public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, string subdomain, int spaceId, string apiKey, string owner, string description) { _activator = activator; _subdomain = subdomain; From 45a7c9039ef15983e04582ce6e67e1d3c1a987e5 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 3 Nov 2025 07:48:44 +0000 Subject: [PATCH 099/142] proep for 9.0.3 release --- CHANGELOG.md | 2 +- Directory.Packages.props | 16 ++++++++-------- .../Dataset/Confluence/ConfluencePageBuilder.cs | 3 +++ rdmp-client.xml | 4 ++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26cf775609..8fb9317711 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [9.0.3] - Unreleased +## [9.0.3] - 2025-11-03 - Improve checking for default pipelines - Improve confluence catalogue metadata extraction diff --git a/Directory.Packages.props b/Directory.Packages.props index a257a13832..311d122b1d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,9 +1,9 @@ - - - - + + + + @@ -22,10 +22,10 @@ - + - - + + @@ -41,7 +41,7 @@ - + diff --git a/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs b/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs index 4e0b4aba3e..2972fdba9c 100644 --- a/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs +++ b/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs @@ -28,6 +28,7 @@ public ConfluencePageBuilder(List catalogues, string name,string desc private string BuildCatalogueOverviewHTML(Catalogue catalogue,string pageId) { + Console.WriteLine($"Building overview for catalogue '{catalogue.Name}'"); string name = pageId != null ?$"{catalogue.Name}" : catalogue.Name; return $""" @@ -60,6 +61,7 @@ Dataset Name } public string BuildContainerPage(Dictionary cataloguePageLookup) { + Console.WriteLine($"Building container page. Found {_catalogues.Count} catalogues."); return $"""

{_repositoryName} Catalogues

@@ -93,6 +95,7 @@ private static string BuildDataVariableRecord(CatalogueItem catalogueItem) public string BuildHTMLForCatalogue(Catalogue catalogue) { + Console.WriteLine($"Building HTML for catalogue '{catalogue.Name}'"); //todo think about splitting and spacing of some of these values return $"""

Dataset Summary

diff --git a/rdmp-client.xml b/rdmp-client.xml index 3e8b773a9a..06b5287257 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -1,7 +1,7 @@ - 9.0.1.0 - https://github.com/HicServices/RDMP/releases/download/v9.0.1/rdmp-9.0.1-client.zip + 9.0.3.0 + https://github.com/HicServices/RDMP/releases/download/v9.0.3/rdmp-9.0.3-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true From 44dd01a8d38078bc3d80c141347ad469ef3eb72e Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 3 Nov 2025 09:15:00 +0000 Subject: [PATCH 100/142] improve project specific handling --- CHANGELOG.md | 1 + ...cuteCommandMakeCatalogueProjectSpecific.cs | 35 ++++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26cf775609..a224dd939d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.0.3] - Unreleased - Improve checking for default pipelines - Improve confluence catalogue metadata extraction +- Improve project specific catalogue error handling ## [9.0.2] - 2025-10-21 - Add ability to view logs as a flat list. Default settings can be updated via user settings diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs index 8ba58e2e04..f8ff6384aa 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs @@ -9,6 +9,7 @@ using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.Providers; using Rdmp.Core.Repositories.Construction; +using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; @@ -29,7 +30,7 @@ public class ExecuteCommandMakeCatalogueProjectSpecific : BasicCommandExecution, [UseWithObjectConstructor] public ExecuteCommandMakeCatalogueProjectSpecific(IBasicActivateItems itemActivator, ICatalogue catalogue, - IProject project, [DemandsInitialization("Ignore Validation",DemandType.Unspecified,defaultValue:false)]bool force) : this(itemActivator) + IProject project, [DemandsInitialization("Ignore Validation", DemandType.Unspecified, defaultValue: false)] bool force) : this(itemActivator) { _catalogue = catalogue; _project = project; @@ -49,14 +50,28 @@ public override string GetCommandHelp() => public override void Execute() { if (_catalogue == null) - SetCatalogue(SelectOne(BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList())); + { + var catalogues = BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); + if (!catalogues.Any()) + { + Show($"No valid catalogues found to make project specific."); + return; + } + SetCatalogue(SelectOne(catalogues)); + } if (!_hasRanCatalogueValidation) { SetCatalogue(_catalogue); } - if(_existingProjectIDs is null) + if (_existingProjectIDs is null) GetExistingProjectIDs(); + var projects = GetListOfValidProjects(); + if (!projects.Any()) + { + Show($"No valid projects found to make {_catalogue.Name} project specific."); + return; + } _project ??= SelectOne(GetListOfValidProjects()); if (_project == null || _catalogue == null) @@ -90,15 +105,17 @@ public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) private List GetListOfValidProjects() { var dataExportChildProvider = ((DataExportChildProvider)_activator.CoreChildProvider); - - var availableProjects = dataExportChildProvider.Projects.Where(p => !dataExportChildProvider.ExtractableDataSetProjects.Where(edsp => edsp.Project_ID == p.ID).Select(edsp => edsp.DataSet.Catalogue).Contains(_catalogue)); - return availableProjects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(_activator.RepositoryLocator.DataExportRepository, _catalogue, p, _existingProjectIDs)).ToList(); + var eds = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWithParent(_catalogue); + var edsp = _activator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(edsp => eds.Contains(edsp.DataSet)); + var pti = edsp.Select(e => e.Project_ID).ToList(); + var validProjects = dataExportChildProvider.Projects.Where(p => !pti.Contains(p.ID) && ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(_activator.RepositoryLocator.DataExportRepository, _catalogue, p, new List())); + return validProjects.ToList(); } private void GetExistingProjectIDs() - { + { var dataExportChildProvider = ((DataExportChildProvider)_activator.CoreChildProvider); - var existingProjects = dataExportChildProvider.Projects.Where(p => dataExportChildProvider.ExtractableDataSetProjects.Where(edsp => edsp.Project_ID==p.ID).Select(edsp => edsp.DataSet.Catalogue).Contains(_catalogue)); + var existingProjects = dataExportChildProvider.Projects.Where(p => dataExportChildProvider.ExtractableDataSetProjects.Where(edsp => edsp.Project_ID == p.ID).Select(edsp => edsp.DataSet.Catalogue).Contains(_catalogue)); _existingProjectIDs = existingProjects.Select(p => p.ID).ToList(); } @@ -113,7 +130,7 @@ private void SetCatalogue(ICatalogue catalogue) return; } var status = _catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository); - if (!GetListOfValidProjects().Any() && !_force) + if (!GetListOfValidProjects().Any() && !_force) { SetImpossible("No valid Projects available"); } From 502ed75ede9dfe96b662c1b56d039dcd78ec97ec Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 3 Nov 2025 11:26:27 +0000 Subject: [PATCH 101/142] update tests --- Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs | 6 +++--- .../ExecuteCommandMakeCatalogueProjectSpecific.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs b/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs index c2d52760ea..0a275681f6 100644 --- a/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs +++ b/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs @@ -51,14 +51,14 @@ public void SetupTests(int projectCount = 0, int projectSpecificCount = 0, int e _activator.Publish(_extractionInfo1); if (projectCount > 0) { - _project1 = new Project(DataExportRepository, "Project1"); + _project1 = new Project(DataExportRepository, $"Project{projectCount}"); _project1.SaveToDatabase(); _activator.Publish(_project1); } if (projectCount > 1) { - _project2 = new Project(DataExportRepository, "Project1"); + _project2 = new Project(DataExportRepository, $"Project{projectCount}"); _project2.SaveToDatabase(); _activator.Publish(_project2); @@ -204,7 +204,7 @@ public void MakeSingleCatalogueProjectSpecificWhenAlreadyInOtherProjectExtractio public void MakeSingleCatalogueProjectSpecificWhenAlreadyInOtherProjectAllowExtractionTest() { SetupTests(2, 0, 1); - MakeProjectSpecific(_catalogue, _project1, new List() { _project2.ID }); + MakeProjectSpecific(_catalogue, _project1, new List() { _project1.ID }); MakeProjectSpecific(_catalogue, _project2, new List() { _project1.ID }); Assert.That(_activator.RepositoryLocator.CatalogueRepository.GetObjectByID(_catalogue.ID).IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository), Is.True); Assert.That(_activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", _catalogue.ID).First().Projects.Count, Is.EqualTo(2)); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs index f8ff6384aa..5a6571843e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs @@ -108,7 +108,7 @@ private List GetListOfValidProjects() var eds = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWithParent(_catalogue); var edsp = _activator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(edsp => eds.Contains(edsp.DataSet)); var pti = edsp.Select(e => e.Project_ID).ToList(); - var validProjects = dataExportChildProvider.Projects.Where(p => !pti.Contains(p.ID) && ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(_activator.RepositoryLocator.DataExportRepository, _catalogue, p, new List())); + var validProjects = dataExportChildProvider.Projects.Where(p => _force ||(!pti.Contains(p.ID) && ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(_activator.RepositoryLocator.DataExportRepository, _catalogue, p, pti))); return validProjects.ToList(); } From df21b52b091598ede5aa34b648cf12f014d15d0b Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 5 Nov 2025 14:47:46 +0000 Subject: [PATCH 102/142] update tests --- ...rtExistingCataloguesIntoExternalDatasetProviderTests.cs | 7 ++++++- SharedAssemblyInfo.cs | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs index 414a1009aa..8d2889b1a2 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs @@ -21,7 +21,8 @@ public void TestImportInternalCataloguesOnly() var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2"); _cata1.SaveToDatabase(); _cata2.SaveToDatabase(); - + var ext = new ExtractableDataSet(GetMockActivator().RepositoryLocator.DataExportRepository, _cata2); + ext.SaveToDatabase(); var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test",1,"test"); var provider = new InternalDatasetProvider(GetMockActivator(),configuration,null); var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false,true, false, false,false); @@ -50,6 +51,10 @@ public void TestImportDeprecatedCataloguesOnly() var _cata1 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset1"); _cata1.IsDeprecated = true; var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2"); + var ext = new ExtractableDataSet(GetMockActivator().RepositoryLocator.DataExportRepository, _cata2); + ext.SaveToDatabase(); + + ext.SaveToDatabase(); _cata1.SaveToDatabase(); _cata2.SaveToDatabase(); diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 7f6033d29f..401bbb05b0 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("9.0.3")] -[assembly: AssemblyFileVersion("9.0.3")] -[assembly: AssemblyInformationalVersion("9.0.3")] +[assembly: AssemblyVersion("9.0.4")] +[assembly: AssemblyFileVersion("9.0.4")] +[assembly: AssemblyInformationalVersion("9.0.4")] From b011c9b729de1d70862689b591f47395ae115176 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 5 Nov 2025 14:48:24 +0000 Subject: [PATCH 103/142] add todo --- ...CommandImportExistingCataloguesIntoExternalDatasetProvider.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs index a3a572722d..e00b815d0d 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -50,6 +50,7 @@ public List GetCatalogues() { catalogues = catalogues.Where(c => !c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList(); } + //todo what to do about non-extractable catalogues???? return catalogues; } From 59e8b8d30beb26140321c4af960427199a65a493 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 6 Nov 2025 08:37:16 +0000 Subject: [PATCH 104/142] [Bugfix] Distinct Selectables (#2235) * distinct selectables * add changelog * bump version --- CHANGELOG.md | 4 ++++ Rdmp.UI/SimpleDialogs/SelectDialog.cs | 4 ++-- SharedAssemblyInfo.cs | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2624438bf..2052c0f96b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.0.4] - Unreleased +- Fix bug with duplicate searchables + ## [9.0.3] - 2025-11-03 - Improve checking for default pipelines - Improve confluence catalogue metadata extraction @@ -11,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.0.2] - 2025-10-21 - Add ability to view logs as a flat list. Default settings can be updated via user settings +- allow the use of Catalogue CIC filters in corresponding Extractions - Add Data Load component to allow SQL to be executed on an external database server - Improve Extraction Log Viewer Filter - Add Dataset Variable document to extractions diff --git a/Rdmp.UI/SimpleDialogs/SelectDialog.cs b/Rdmp.UI/SimpleDialogs/SelectDialog.cs index affb50df05..b02f7f82e6 100644 --- a/Rdmp.UI/SimpleDialogs/SelectDialog.cs +++ b/Rdmp.UI/SimpleDialogs/SelectDialog.cs @@ -169,7 +169,7 @@ public SelectDialog(DialogArgs args, IActivateItems activator, IEnumerable to if (IsDatabaseObjects()) { - _allObjects = toSelectFrom.ToArray(); + _allObjects = toSelectFrom.Distinct().ToArray(); _searchables = _allObjects.Cast() .ToDictionary(k => k, activator.CoreChildProvider.GetDescendancyListIfAnyFor); _usefulPropertyFinder = new AttributePropertyFinder(_searchables.Keys); @@ -188,7 +188,7 @@ public SelectDialog(DialogArgs args, IActivateItems activator, IEnumerable to } else { - _allObjects = toSelectFrom.ToArray(); + _allObjects = toSelectFrom.Distinct().ToArray(); // don't bother with the tool strip because its not database objects so we can't filter by ID/type etc Controls.Remove(toolStrip1); diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 7f6033d29f..401bbb05b0 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("9.0.3")] -[assembly: AssemblyFileVersion("9.0.3")] -[assembly: AssemblyInformationalVersion("9.0.3")] +[assembly: AssemblyVersion("9.0.4")] +[assembly: AssemblyFileVersion("9.0.4")] +[assembly: AssemblyInformationalVersion("9.0.4")] From b0b004b14170fa6df49a1210faf3e19acaaa4bab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 08:38:27 +0000 Subject: [PATCH 105/142] Bump actions/setup-dotnet from 4.3.1 to 5.0.0 (#2230) Bumps [actions/setup-dotnet](https://github.com/actions/setup-dotnet) from 4.3.1 to 5.0.0. - [Release notes](https://github.com/actions/setup-dotnet/releases) - [Commits](https://github.com/actions/setup-dotnet/compare/v4.3.1...v5.0.0) --- updated-dependencies: - dependency-name: actions/setup-dotnet dependency-version: 5.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- .github/workflows/docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 21f8bc3df3..41ad40cdcd 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: setup .NET - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 with: global-json-file: global.json dotnet-version: '6.0.x' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 33c0c80c72..3384247a7f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: with: submodules: true - name: Setup .NET Core - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 with: dotnet-version: 6.0.x - name: Cache Nuget From af171d9dedd4b245ed3698abdd02dfe46d8c5600 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 11:28:18 +0000 Subject: [PATCH 106/142] Bump github/codeql-action from 3 to 4 (#2241) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3 to 4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3...v4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 41ad40cdcd..28e7580d8a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -32,7 +32,7 @@ jobs: dotnet-version: '6.0.x' - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} queries: +security-and-quality @@ -49,13 +49,13 @@ jobs: sed -i 's/"language": ""/"language": "en-US"/' sarif-results/scs.sarif - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{ matrix.language }}" upload: False output: sarif-results - name: Upload SARIF - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: sarif-results From eee3bf5f45c90ffcf5241381d3b75d0ce3847210 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 11:28:34 +0000 Subject: [PATCH 107/142] Bump shogo82148/actions-setup-perl from 1.34.0 to 1.36.0 (#2247) --- updated-dependencies: - dependency-name: shogo82148/actions-setup-perl dependency-version: 1.36.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 726b61e76b..e2b45ae2b3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -216,7 +216,7 @@ jobs: - name: Install Perl dependencies - uses: shogo82148/actions-setup-perl@v1.34.0 + uses: shogo82148/actions-setup-perl@v1.36.0 with: install-modules-with: cpanm install-modules: Archive::Zip Archive::Tar From b7228db3fb2194a0026bb00630c954048e0544e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 11:28:52 +0000 Subject: [PATCH 108/142] Bump actions/checkout from 4 to 5 (#2226) Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 6 +++--- .github/workflows/codeql.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/links.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e2b45ae2b3..39c5aae7e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: runs-on: windows-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: setup .NET uses: actions/setup-dotnet@v5.0.0 - name: Populate Databases.yaml @@ -84,7 +84,7 @@ jobs: runs-on: windows-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: setup .NET uses: actions/setup-dotnet@v5.0.0 - name: Populate Databases.yaml @@ -152,7 +152,7 @@ jobs: runs-on: windows-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: setup .NET uses: actions/setup-dotnet@v5.0.0 - name: Determine RDMP build version diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 28e7580d8a..cff7cefa04 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: setup .NET uses: actions/setup-dotnet@v5.0.0 with: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3384247a7f..b85738f672 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,7 +13,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: submodules: true - name: Setup .NET Core diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index e149d9ac23..2f993feb21 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: submodules: recursive - name: Run link check From b5a71a5ab18be02426950aa70814b4351d3f5c49 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 6 Nov 2025 11:45:24 +0000 Subject: [PATCH 109/142] add flat view toggle --- CHANGELOG.md | 1 + .../Settings/UserSettings.cs | 6 +- .../CatalogueCollectionFilterUI.Designer.cs | 103 ++++++++++-------- .../CatalogueCollectionFilterUI.cs | 3 +- .../CatalogueCollectionFilterUI.resx | 54 ++++----- .../CatalogueCollectionUI.Designer.cs | 24 ++-- Rdmp.UI/Collections/CatalogueCollectionUI.cs | 45 +++++++- .../Collections/CatalogueCollectionUI.resx | 6 +- 8 files changed, 151 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2052c0f96b..7551acfcb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.0.4] - Unreleased - Fix bug with duplicate searchables +- Introduce ability to view Catalogues in a flat view ## [9.0.3] - 2025-11-03 - Improve checking for default pipelines diff --git a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs index 82cfe307ca..541c79a602 100644 --- a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs +++ b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs @@ -27,7 +27,11 @@ public static class UserSettings throw new NotImplementedException( "Isolated Storage does not work in this environment..."); - + public static bool ShowFlatLists + { + get => AppSettings.GetValueOrDefault("ShowFlatLists", true); + set => AppSettings.AddOrUpdateValue("ShowFlatLists", value); + } public static bool UseLocalFileSystem { get => AppSettings.GetValueOrDefault("UseLocalFileSystem", false); diff --git a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs index 7300d0879c..93c5a626c6 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs @@ -28,65 +28,79 @@ protected override void Dispose(bool disposing) /// private void InitializeComponent() { - this.cbShowDeprecated = new System.Windows.Forms.CheckBox(); - this.cbShowInternal = new System.Windows.Forms.CheckBox(); - this.cbProjectSpecific = new System.Windows.Forms.CheckBox(); - this.flowLayoutPanel1 = new System.Windows.Forms.FlowLayoutPanel(); - this.flowLayoutPanel1.SuspendLayout(); - this.SuspendLayout(); + cbShowDeprecated = new System.Windows.Forms.CheckBox(); + cbShowInternal = new System.Windows.Forms.CheckBox(); + cbProjectSpecific = new System.Windows.Forms.CheckBox(); + flowLayoutPanel1 = new System.Windows.Forms.FlowLayoutPanel(); + cbFlatView = new System.Windows.Forms.CheckBox(); + flowLayoutPanel1.SuspendLayout(); + SuspendLayout(); // // cbShowDeprecated // - this.cbShowDeprecated.AutoSize = true; - this.cbShowDeprecated.Location = new System.Drawing.Point(163, 3); - this.cbShowDeprecated.Name = "cbShowDeprecated"; - this.cbShowDeprecated.Size = new System.Drawing.Size(82, 17); - this.cbShowDeprecated.TabIndex = 6; - this.cbShowDeprecated.Text = "Deprecated"; - this.cbShowDeprecated.UseVisualStyleBackColor = true; - this.cbShowDeprecated.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); + cbShowDeprecated.AutoSize = true; + cbShowDeprecated.Location = new System.Drawing.Point(75, 3); + cbShowDeprecated.Name = "cbShowDeprecated"; + cbShowDeprecated.Size = new System.Drawing.Size(86, 19); + cbShowDeprecated.TabIndex = 6; + cbShowDeprecated.Text = "Deprecated"; + cbShowDeprecated.UseVisualStyleBackColor = true; + cbShowDeprecated.CheckedChanged += OnCheckboxChanged; // // cbShowInternal // - this.cbShowInternal.AutoSize = true; - this.cbShowInternal.Location = new System.Drawing.Point(96, 3); - this.cbShowInternal.Name = "cbShowInternal"; - this.cbShowInternal.Size = new System.Drawing.Size(61, 17); - this.cbShowInternal.TabIndex = 7; - this.cbShowInternal.Text = "Internal"; - this.cbShowInternal.UseVisualStyleBackColor = true; - this.cbShowInternal.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); + cbShowInternal.AutoSize = true; + cbShowInternal.Location = new System.Drawing.Point(3, 3); + cbShowInternal.Name = "cbShowInternal"; + cbShowInternal.Size = new System.Drawing.Size(66, 19); + cbShowInternal.TabIndex = 7; + cbShowInternal.Text = "Internal"; + cbShowInternal.UseVisualStyleBackColor = true; + cbShowInternal.CheckedChanged += OnCheckboxChanged; // // cbProjectSpecific // - this.cbProjectSpecific.AutoSize = true; - this.cbProjectSpecific.Location = new System.Drawing.Point(3, 26); - this.cbProjectSpecific.Name = "cbProjectSpecific"; - this.cbProjectSpecific.Size = new System.Drawing.Size(100, 17); - this.cbProjectSpecific.TabIndex = 8; - this.cbProjectSpecific.Text = "Project Specific"; - this.cbProjectSpecific.UseVisualStyleBackColor = true; - this.cbProjectSpecific.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); + cbProjectSpecific.AutoSize = true; + cbProjectSpecific.Location = new System.Drawing.Point(3, 28); + cbProjectSpecific.Name = "cbProjectSpecific"; + cbProjectSpecific.Size = new System.Drawing.Size(107, 19); + cbProjectSpecific.TabIndex = 8; + cbProjectSpecific.Text = "Project Specific"; + cbProjectSpecific.UseVisualStyleBackColor = true; + cbProjectSpecific.CheckedChanged += OnCheckboxChanged; // // flowLayoutPanel1 // - this.flowLayoutPanel1.Controls.Add(this.cbShowInternal); - this.flowLayoutPanel1.Controls.Add(this.cbShowDeprecated); - this.flowLayoutPanel1.Controls.Add(this.cbProjectSpecific); - this.flowLayoutPanel1.Dock = System.Windows.Forms.DockStyle.Fill; - this.flowLayoutPanel1.Location = new System.Drawing.Point(0, 0); - this.flowLayoutPanel1.Name = "flowLayoutPanel1"; - this.flowLayoutPanel1.Size = new System.Drawing.Size(256, 69); - this.flowLayoutPanel1.TabIndex = 10; + flowLayoutPanel1.Controls.Add(cbShowInternal); + flowLayoutPanel1.Controls.Add(cbShowDeprecated); + flowLayoutPanel1.Controls.Add(cbProjectSpecific); + flowLayoutPanel1.Controls.Add(cbFlatView); + flowLayoutPanel1.Dock = System.Windows.Forms.DockStyle.Fill; + flowLayoutPanel1.Location = new System.Drawing.Point(0, 0); + flowLayoutPanel1.Name = "flowLayoutPanel1"; + flowLayoutPanel1.Size = new System.Drawing.Size(256, 69); + flowLayoutPanel1.TabIndex = 10; + // + // cbFlatView + // + cbFlatView.Appearance = System.Windows.Forms.Appearance.Button; + cbFlatView.AutoSize = true; + cbFlatView.Location = new System.Drawing.Point(116, 28); + cbFlatView.Name = "cbFlatView"; + cbFlatView.Size = new System.Drawing.Size(64, 25); + cbFlatView.TabIndex = 9; + cbFlatView.Text = "Flat View"; + cbFlatView.UseVisualStyleBackColor = true; + cbFlatView.CheckedChanged += OnCheckboxChanged; // // CatalogueCollectionFilterUI // - this.Controls.Add(this.flowLayoutPanel1); - this.Name = "CatalogueCollectionFilterUI"; - this.Size = new System.Drawing.Size(256, 69); - this.flowLayoutPanel1.ResumeLayout(false); - this.flowLayoutPanel1.PerformLayout(); - this.ResumeLayout(false); + Controls.Add(flowLayoutPanel1); + Name = "CatalogueCollectionFilterUI"; + Size = new System.Drawing.Size(256, 69); + flowLayoutPanel1.ResumeLayout(false); + flowLayoutPanel1.PerformLayout(); + ResumeLayout(false); } @@ -96,5 +110,6 @@ private void InitializeComponent() private System.Windows.Forms.CheckBox cbShowInternal; private System.Windows.Forms.CheckBox cbProjectSpecific; private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel1; + private System.Windows.Forms.CheckBox cbFlatView; } } diff --git a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs index 50e7c08365..4c1f653f92 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs @@ -22,6 +22,7 @@ public CatalogueCollectionFilterUI() cbShowInternal.Checked = UserSettings.ShowInternalCatalogues; cbShowDeprecated.Checked = UserSettings.ShowDeprecatedCatalogues; cbProjectSpecific.Checked = UserSettings.ShowProjectSpecificCatalogues; + cbFlatView.Checked = UserSettings.ShowFlatLists; _loading = false; } @@ -36,7 +37,7 @@ private void OnCheckboxChanged(object sender, EventArgs e) UserSettings.ShowInternalCatalogues = cbShowInternal.Checked; UserSettings.ShowDeprecatedCatalogues = cbShowDeprecated.Checked; UserSettings.ShowProjectSpecificCatalogues = cbProjectSpecific.Checked; - + UserSettings.ShowFlatLists = cbFlatView.Checked; FiltersChanged?.Invoke(this, EventArgs.Empty); } diff --git a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.resx b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.resx index 1af7de150c..8b2ff64a11 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.resx +++ b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.resx @@ -1,17 +1,17 @@  - diff --git a/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs b/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs index 639afdee97..bb5ee10b07 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs @@ -30,9 +30,9 @@ private void InitializeComponent() olvOrder = new OLVColumn(); imageList_RightClickIcons = new ImageList(components); gbCatalogueFilters = new GroupBox(); + tbFilter = new TextBox(); catalogueCollectionFilterUI1 = new CatalogueCollectionFilterUI(); panel2 = new Panel(); - tbFilter = new TextBox(); ((ISupportInitialize)tlvCatalogues).BeginInit(); gbCatalogueFilters.SuspendLayout(); panel2.SuspendLayout(); @@ -49,7 +49,7 @@ private void InitializeComponent() tlvCatalogues.Location = new System.Drawing.Point(0, 0); tlvCatalogues.Name = "tlvCatalogues"; tlvCatalogues.ShowGroups = false; - tlvCatalogues.Size = new System.Drawing.Size(500, 414); + tlvCatalogues.Size = new System.Drawing.Size(500, 403); tlvCatalogues.TabIndex = 0; tlvCatalogues.Text = "label1"; tlvCatalogues.UseCompatibleStateImageBehavior = false; @@ -95,19 +95,26 @@ private void InitializeComponent() gbCatalogueFilters.Controls.Add(tbFilter); gbCatalogueFilters.Controls.Add(catalogueCollectionFilterUI1); gbCatalogueFilters.Dock = DockStyle.Bottom; - gbCatalogueFilters.Location = new System.Drawing.Point(0, 414); + gbCatalogueFilters.Location = new System.Drawing.Point(0, 403); gbCatalogueFilters.Name = "gbCatalogueFilters"; - gbCatalogueFilters.Size = new System.Drawing.Size(500, 65); + gbCatalogueFilters.Size = new System.Drawing.Size(500, 76); gbCatalogueFilters.TabIndex = 1; gbCatalogueFilters.TabStop = false; gbCatalogueFilters.Text = "Show"; // + // tbFilter + // + tbFilter.Location = new System.Drawing.Point(0, 53); + tbFilter.Name = "tbFilter"; + tbFilter.Size = new System.Drawing.Size(500, 23); + tbFilter.TabIndex = 2; + // // catalogueCollectionFilterUI1 // catalogueCollectionFilterUI1.Dock = DockStyle.Fill; catalogueCollectionFilterUI1.Location = new System.Drawing.Point(3, 19); catalogueCollectionFilterUI1.Name = "catalogueCollectionFilterUI1"; - catalogueCollectionFilterUI1.Size = new System.Drawing.Size(494, 43); + catalogueCollectionFilterUI1.Size = new System.Drawing.Size(494, 54); catalogueCollectionFilterUI1.TabIndex = 0; // // panel2 @@ -120,13 +127,6 @@ private void InitializeComponent() panel2.Size = new System.Drawing.Size(500, 479); panel2.TabIndex = 2; // - // tbFilter - // - tbFilter.Location = new System.Drawing.Point(0, 39); - tbFilter.Name = "tbFilter"; - tbFilter.Size = new System.Drawing.Size(500, 23); - tbFilter.TabIndex = 2; - // // CatalogueCollectionUI // Controls.Add(panel2); diff --git a/Rdmp.UI/Collections/CatalogueCollectionUI.cs b/Rdmp.UI/Collections/CatalogueCollectionUI.cs index 7a1c34deb4..18a6c30136 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionUI.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionUI.cs @@ -16,6 +16,7 @@ using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.Providers.Nodes; +using Rdmp.Core.ReusableLibraryCode.Settings; using Rdmp.UI.Collections.Providers.Filtering; using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.ItemActivation; @@ -49,6 +50,8 @@ public partial class CatalogueCollectionUI : RDMPCollectionUI private bool bLoading = true; + private bool _renderingFlatFiew = false; + public CatalogueCollectionUI() { InitializeComponent(); @@ -107,8 +110,19 @@ public void RefreshUIFromDatabase(object oRefreshFrom) var newCatalogues = CommonTreeFunctionality.CoreChildProvider.AllCatalogues.Except(_allCatalogues); if (newCatalogues.Any()) { - oRefreshFrom = rootFolder; //refresh from the root instead - tlvCatalogues.RefreshObject(oRefreshFrom); + if (UserSettings.ShowFlatLists) + { + oRefreshFrom = null; + tlvCatalogues.AddObjects(newCatalogues.ToList()); + tlvCatalogues.RefreshObjects(Activator.CoreChildProvider.AllCatalogues); + _renderingFlatFiew = true; + } + else + { + oRefreshFrom = rootFolder; //refresh from the root instead + tlvCatalogues.RefreshObject(oRefreshFrom); + _renderingFlatFiew = false; + } } } @@ -184,6 +198,22 @@ public void ApplyFilters() tlvCatalogues.UseFiltering = true; tlvCatalogues.ModelFilter = new CatalogueCollectionFilter(Activator.CoreChildProvider); + if (UserSettings.ShowFlatLists && !_renderingFlatFiew) + { + //reset to flat view + tlvCatalogues.RemoveObject(Activator.CoreChildProvider.CatalogueRootFolder); + tlvCatalogues.AddObjects(Activator.CoreChildProvider.AllCatalogues); + //tlvCatalogues.RefreshObjects(Activator.CoreChildProvider.AllCatalogues); + _renderingFlatFiew = true; + } + else if(!UserSettings.ShowFlatLists && _renderingFlatFiew) + { + //reset to folder view + tlvCatalogues.RemoveObjects(Activator.CoreChildProvider.AllCatalogues); + tlvCatalogues.AddObject(Activator.CoreChildProvider.CatalogueRootFolder); + tlvCatalogues.RefreshObject(Activator.CoreChildProvider.CatalogueRootFolder); + _renderingFlatFiew = false; + } } public enum HighlightCatalogueType @@ -259,7 +289,16 @@ public override void SetItemActivator(IActivateItems activator) Activator.RefreshBus.EstablishLifetimeSubscription(this); tlvCatalogues.AddObject(activator.CoreChildProvider.AllGovernanceNode); - tlvCatalogues.AddObject(activator.CoreChildProvider.CatalogueRootFolder); + if (UserSettings.ShowFlatLists) + { + tlvCatalogues.AddObjects(activator.CoreChildProvider.AllCatalogues); + _renderingFlatFiew = true; + } + else + { + tlvCatalogues.AddObject(activator.CoreChildProvider.CatalogueRootFolder); + _renderingFlatFiew = false; + } ApplyFilters(); RefreshUIFromDatabase(activator.CoreChildProvider.CatalogueRootFolder); diff --git a/Rdmp.UI/Collections/CatalogueCollectionUI.resx b/Rdmp.UI/Collections/CatalogueCollectionUI.resx index 4e69981475..1dc9e525a6 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionUI.resx +++ b/Rdmp.UI/Collections/CatalogueCollectionUI.resx @@ -1,7 +1,7 @@  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - - ..\zh-Hans\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + ..\zh-Hans\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + + + + \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/StateBasedIconProviders/CohortIdentificationConfigurationStateBasedIconProvider.cs b/Rdmp.Core/Icons/IconProvision/StateBasedIconProviders/CohortIdentificationConfigurationStateBasedIconProvider.cs index 4729c49418..0703ceb6ba 100644 --- a/Rdmp.Core/Icons/IconProvision/StateBasedIconProviders/CohortIdentificationConfigurationStateBasedIconProvider.cs +++ b/Rdmp.Core/Icons/IconProvision/StateBasedIconProviders/CohortIdentificationConfigurationStateBasedIconProvider.cs @@ -18,6 +18,7 @@ public class CohortIdentificationConfigurationStateBasedIconProvider : IObjectSt private readonly Image _cohortIdentificationConfigurationVersion; private readonly Image _frozenCohortIdentificationConfiguration; private readonly Image _frozenCohortIdentificationConfigurationVersion; + private readonly Image _templateCogortIdentificationConfiguration; public CohortIdentificationConfigurationStateBasedIconProvider() { @@ -26,13 +27,14 @@ public CohortIdentificationConfigurationStateBasedIconProvider() _frozenCohortIdentificationConfiguration = Image.Load(CatalogueIcons.FrozenCohortIdentificationConfiguration); _frozenCohortIdentificationConfigurationVersion = IconOverlayProvider.GetOverlay(_frozenCohortIdentificationConfiguration, OverlayKind.Version); + _templateCogortIdentificationConfiguration = Image.Load(CatalogueIcons.TemplateCohortIdentificationConfiguration); } public Image GetImageIfSupportedObject(object o) => o is not CohortIdentificationConfiguration cic ? null - : cic.Frozen + : cic.IsTemplate?_templateCogortIdentificationConfiguration:cic.Frozen ? cic.Version is null?_frozenCohortIdentificationConfiguration: _frozenCohortIdentificationConfigurationVersion : cic.Version is null ? _cohortIdentificationConfiguration: _cohortIdentificationConfigurationVersion; } \ No newline at end of file diff --git a/Rdmp.Core/Icons/TemplateCohortIdentificationConfiguration.png b/Rdmp.Core/Icons/TemplateCohortIdentificationConfiguration.png new file mode 100644 index 0000000000000000000000000000000000000000..c263543476e25a05d266e6ed43edd22bf0425670 GIT binary patch literal 1015 zcmVN2bZe?^J zG%heMF*7s+TY3Nh19VA5K~y+TjZ|G|TtyWA=Kl2F-E7;Wq+n4~t%X=CEul5?6pI7?g@ESCd zHIn>#diMIFQYobRJD=}~W4L{y>u0ldYwliV9Qpk6@%>-muXszMMka}d>~K(bo#exX z$u3jVO@k2VOyx-d_CNMaTI4JZcUweNG3SthT?fW9B36SETS=tZ_fxH<=|@`3gqffW z8#EJbW&&+S0+(lgw?y7*X$2wEkSNdhj(ucomxp5rvU+(&X>L=>p6yV4A1Re!BUjL> zYS=`CjdC6zxS1~HXQ0#K>jBlCjiM21%6=0QQ&i>$0oM5r3|^W^U96{7Y+x=@VSsB> zm!vv{|0*80D&h0LkYq!pa0DTD_)xhDMKyrH1@SA;d>bu61xm=*VeKq0M%;GDnz|uU z#^;ca>gLXc!scC*0ENeiT+fE%m63OCl=(UcBC^&kJCJ;6$FYDqKgvK1Nj8{LhJ+KU zM&Rn}SMBm5Jk3Cv`Y3S10w-GKM2=9;QCzr*4sU#F-*GrmB!b|mOJ$zlvbyYTy;W?( zO4XL5#49R7heLGqJpGDnkJ0`I_0k9DtRF5 zO5>dfS5Lo__CuJ2uy(PIUXww2Fj0UoAG!GzoV}F8xrtfmN{HS4UpIX;^Ua|&ff8_+ zNM)2VmRKj&%?NxK37z8BOaY_U=6IsV8`sCqzAu{y?_B)&AUBeXd9-@NpiW6_h`XxY zm#H(D8pFmUL3>g~QUfRF!%(rO|EmYDh1MIkz5DRd?N4_0c6Ou?@V}o34e!H5`-^xK l(=-*?J2oax9*iFZ Date: Mon, 10 Nov 2025 14:25:46 +0000 Subject: [PATCH 114/142] add use commands --- .../CommandExecution/AtomicCommandFactory.cs | 24 +++++++++-- ...hortIdentificationConfigurationTemplate.cs | 43 +++++++++++++++++++ ...mplateCohortIdentificationConfiguration.cs | 43 +++++++++++++++++++ Rdmp.Core/Providers/CatalogueChildProvider.cs | 5 +-- ...eCohortIdentificationConfigurationsNode.cs | 9 ++-- .../CohortIdentificationCollectionUI.cs | 10 +---- 6 files changed, 114 insertions(+), 20 deletions(-) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index f1c1bf3861..9a4070f8b9 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -478,10 +478,26 @@ public IEnumerable CreateCommands(object o) yield return new ExecuteCommandViewData(_activator, cic, ViewType.All, null, true) { Weight = -99.7f }; yield return new ExecuteCommandViewData(_activator, cic, ViewType.All, null, false) { Weight = -99.6f }; - yield return new ExecuteCommandFreezeCohortIdentificationConfiguration(_activator, cic, !cic.Frozen) - { Weight = -50.5f }; - yield return new ExecuteCommandCreateHoldoutLookup(_activator, cic) - { Weight = -50.5f }; + if (cic.IsTemplate) + { + yield return new ExecuteCommandUseTemplateCohortIdentificationConfiguration(_activator, cic) + { + Weight = -50.5f + }; + } + else + { + yield return new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(_activator, cic) + { + Weight = -50.5f + }; + yield return new ExecuteCommandFreezeCohortIdentificationConfiguration(_activator, cic, !cic.Frozen) + { Weight = -50.5f }; + yield return new ExecuteCommandCreateHoldoutLookup(_activator, cic) + { Weight = -50.5f }; + } + + var clone = new ExecuteCommandCloneCohortIdentificationConfiguration(_activator) { Weight = -50.4f, OverrideCommandName = "Clone" }.SetTarget(cic); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs new file mode 100644 index 0000000000..ac0e6265cb --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs @@ -0,0 +1,43 @@ +using NPOI.OpenXmlFormats.Encryption; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.ReusableLibraryCode.Checks; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands +{ + public class ExecuteCommandCreateCohortIdentificationConfigurationTemplate: BasicCommandExecution,IAtomicCommandWithTarget + { + private CohortIdentificationConfiguration _cic; + private IBasicActivateItems _activator; + public ExecuteCommandCreateCohortIdentificationConfigurationTemplate(IBasicActivateItems activator, CohortIdentificationConfiguration cic): base(activator) + { + _activator = activator; + _cic = cic; + } + + public override void Execute() + { + base.Execute(); + var clone = _cic.CreateClone(ThrowImmediatelyCheckNotifier.Quiet); + clone.IsTemplate = true; + clone.Name = clone.Name.Replace("(Clone)", "") + " Template"; + clone.SaveToDatabase(); + Publish(clone); + } + + public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) + { + if(target is not CohortIdentificationConfiguration) + { + throw new Exception("Provided database entity was not a CohortIdentificationConfiguration."); + } + _cic = target as CohortIdentificationConfiguration; + return this; + } + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs new file mode 100644 index 0000000000..68cc0d54f3 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs @@ -0,0 +1,43 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.ReusableLibraryCode.Checks; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands +{ + public class ExecuteCommandUseTemplateCohortIdentificationConfiguration : BasicCommandExecution, IAtomicCommandWithTarget + { + //todo will want to think about how to associate this with a project + private CohortIdentificationConfiguration _cic; + private IBasicActivateItems _activator; + public ExecuteCommandUseTemplateCohortIdentificationConfiguration(IBasicActivateItems activator, CohortIdentificationConfiguration cic) : base(activator) + { + _activator = activator; + _cic = cic; + } + + public override void Execute() + { + base.Execute(); + var clone = _cic.CreateClone(ThrowImmediatelyCheckNotifier.Quiet); + clone.IsTemplate = false; + clone.Name = clone.Name.Replace(" Template", ""); + clone.SaveToDatabase(); + Publish(clone); + } + + public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) + { + if (target is not CohortIdentificationConfiguration cic || !cic.IsTemplate) + { + throw new Exception("Provided database entity was not a CohortIdentificationConfiguration."); + } + _cic = target as CohortIdentificationConfiguration; + return this; + } + } +} diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 3f826313ac..248ea43270 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -5,6 +5,7 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using System; +using System.ClientModel.Primitives; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; @@ -1544,8 +1545,6 @@ protected void AddToDictionaries(HashSet children, DescendancyList list) //document that the last parent has these as children var parent = list.Last(); - var x = _childDictionary.TryGetValue(parent, out var foundchild); - _childDictionary.AddOrUpdate(parent, children, (p, s) => children); @@ -1590,7 +1589,7 @@ public virtual object[] GetChildren(object model) { lock (WriteLock) { - //if we have a record of any children in the child dictionary for the parent model object +; //if we have a record of any children in the child dictionary for the parent model object if (_childDictionary.TryGetValue(model, out var cached)) return cached.OrderBy(static o => o.ToString()).ToArray(); diff --git a/Rdmp.Core/Providers/Nodes/CohortNodes/AllTemplateCohortIdentificationConfigurationsNode.cs b/Rdmp.Core/Providers/Nodes/CohortNodes/AllTemplateCohortIdentificationConfigurationsNode.cs index 36093149f1..ed40f2318a 100644 --- a/Rdmp.Core/Providers/Nodes/CohortNodes/AllTemplateCohortIdentificationConfigurationsNode.cs +++ b/Rdmp.Core/Providers/Nodes/CohortNodes/AllTemplateCohortIdentificationConfigurationsNode.cs @@ -1,4 +1,5 @@ -using System; +using Rdmp.Core.DataExport.Data; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -6,12 +7,10 @@ namespace Rdmp.Core.Providers.Nodes.CohortNodes { - public class AllTemplateCohortIdentificationConfigurationsNode : Node + public class AllTemplateCohortIdentificationConfigurationsNode : SingletonNode { - public AllTemplateCohortIdentificationConfigurationsNode() + public AllTemplateCohortIdentificationConfigurationsNode() : base("Template Cohort Configurations") { } - public override string ToString() => "Template Cohort Configurations"; - } } diff --git a/Rdmp.UI/Collections/CohortIdentificationCollectionUI.cs b/Rdmp.UI/Collections/CohortIdentificationCollectionUI.cs index 33c06204f0..3d03f2f1e8 100644 --- a/Rdmp.UI/Collections/CohortIdentificationCollectionUI.cs +++ b/Rdmp.UI/Collections/CohortIdentificationCollectionUI.cs @@ -60,7 +60,7 @@ public override void SetItemActivator(IActivateItems activator) CommonTreeFunctionality.MaintainRootObjects = new[] { typeof(FolderNode), - typeof(FolderNode), + typeof(AllTemplateCohortIdentificationConfigurationsNode), typeof(AllOrphanAggregateConfigurationsNode), typeof(AllTemplateAggregateConfigurationsNode) }; @@ -68,13 +68,7 @@ public override void SetItemActivator(IActivateItems activator) rootFolder.ChildFolders = new List>(); rootFolder.ChildObjects = new List(); - //var templateRootFolder = Activator.CoreChildProvider.TemplateCohortIdentificationConfigurationRootFolder; - //templateRootFolder.Name = "Templates"; - //templateRootFolder.ChildFolders = new List>(); - //templateRootFolder.ChildObjects = new List(); - tlvCohortIdentificationConfigurations.AddObject(rootFolder); - //tlvCohortIdentificationConfigurations.AddObject(templateRootFolder); tlvCohortIdentificationConfigurations.AddObject(Activator.CoreChildProvider.AllTemplateCohortIdentificationConfigurationsNode); tlvCohortIdentificationConfigurations.AddObject(Activator.CoreChildProvider.OrphanAggregateConfigurationsNode); tlvCohortIdentificationConfigurations.AddObject(Activator.CoreChildProvider @@ -149,7 +143,7 @@ public static bool IsRootObject(object root) => // The root CohortIdentificationConfiguration FolderNode is a root element in this tree root is FolderNode f ? f.Name == FolderHelper.Root - : root is AllOrphanAggregateConfigurationsNode or AllTemplateAggregateConfigurationsNode; + : root is AllOrphanAggregateConfigurationsNode or AllTemplateAggregateConfigurationsNode or AllTemplateCohortIdentificationConfigurationsNode; public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) { From 258ccb75c12cce6bfe8010a6e6a4eded0fd268b8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 10 Nov 2025 16:16:47 +0000 Subject: [PATCH 115/142] improved project template nodes --- .../CommandExecution/AtomicCommandFactory.cs | 91 +++++------ ...hortIdentificationConfigurationTemplate.cs | 35 +++++ ...mplateCohortIdentificationConfiguration.cs | 42 +++++- Rdmp.Core/DataExport/Data/Project.cs | 9 +- ...iatedCohortIdentificationTemplatesNode.png | Bin 0 -> 1149 bytes .../Icons/IconProvision/CatalogueIcons.resx | 9 +- .../IconProvision/CatalogueIcons.zh-Hans.resx | 141 +++++++++--------- Rdmp.Core/Icons/IconProvision/RDMPConcept.cs | 3 +- .../Providers/DataExportChildProvider.cs | 18 +++ ...ciatedCohortIdentificationTemplatesNode.cs | 40 +++++ .../CohortIdentificationConfigurationUI.cs | 20 ++- 11 files changed, 282 insertions(+), 126 deletions(-) create mode 100644 Rdmp.Core/Icons/AssociatedCohortIdentificationTemplatesNode.png create mode 100644 Rdmp.Core/Providers/Nodes/CohortNodes/AssociatedCohortIdentificationTemplatesNode.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 9a4070f8b9..d621b3548d 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -465,15 +465,7 @@ public IEnumerable CreateCommands(object o) { if (pcic != null) cic = pcic.CohortIdentificationConfiguration; - var commit = - new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration(_activator, null) - { - OverrideCommandName = "Commit Cohort", - Weight = -99.8f - }.SetTarget(cic); - if (pcic != null) commit.SetTarget((DatabaseEntity)pcic.Project); - yield return commit; yield return new ExecuteCommandViewData(_activator, cic, ViewType.All, null, true) { Weight = -99.7f }; yield return new ExecuteCommandViewData(_activator, cic, ViewType.All, null, false) { Weight = -99.6f }; @@ -487,6 +479,15 @@ public IEnumerable CreateCommands(object o) } else { + var commit = + new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration(_activator, null) + { + OverrideCommandName = "Commit Cohort", + Weight = -99.8f + }.SetTarget(cic); + if (pcic != null) commit.SetTarget((DatabaseEntity)pcic.Project); + + yield return commit; yield return new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(_activator, cic) { Weight = -50.5f @@ -495,20 +496,20 @@ public IEnumerable CreateCommands(object o) { Weight = -50.5f }; yield return new ExecuteCommandCreateHoldoutLookup(_activator, cic) { Weight = -50.5f }; + var clone = new ExecuteCommandCloneCohortIdentificationConfiguration(_activator) + { Weight = -50.4f, OverrideCommandName = "Clone" }.SetTarget(cic); + if (pcic != null) clone.SetTarget((DatabaseEntity)pcic.Project); + yield return clone; + + yield return new ExecuteCommandSetQueryCachingDatabase(_activator, cic) + { Weight = -50.4f, OverrideCommandName = "Change Query Cache" }; } - - var clone = new ExecuteCommandCloneCohortIdentificationConfiguration(_activator) - { Weight = -50.4f, OverrideCommandName = "Clone" }.SetTarget(cic); - if (pcic != null) clone.SetTarget((DatabaseEntity)pcic.Project); - yield return clone; //associate with project yield return new ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(_activator) { Weight = -50.3f, OverrideCommandName = "Associate with Project" }.SetTarget(cic); - yield return new ExecuteCommandSetQueryCachingDatabase(_activator, cic) - { Weight = -50.4f, OverrideCommandName = "Change Query Cache" }; } if (Is(o, out AllGovernanceNode _)) @@ -941,35 +942,39 @@ public IEnumerable CreateCommands(object o) if (Is(o, out CohortAggregateContainer cohortAggregateContainer)) { - yield return new ExecuteCommandAddCatalogueToCohortIdentificationSetContainer(_activator, - cohortAggregateContainer, null, null) - { SuggestedCategory = Add, OverrideCommandName = "Catalogue" }; - yield return new ExecuteCommandAddCohortSubContainer(_activator, cohortAggregateContainer) - { SuggestedCategory = Add, OverrideCommandName = "Sub Container" }; - yield return new ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer(_activator, - cohortAggregateContainer, true) - { SuggestedCategory = Add, OverrideCommandName = "Existing Cohort Set (copy of)" }; - yield return new ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer(_activator, - cohortAggregateContainer, false) - { SuggestedCategory = Add, OverrideCommandName = "Aggregate" }; - yield return new ExecuteCommandImportCohortIdentificationConfiguration(_activator, null, + var associatedCIC = cohortAggregateContainer.GetCohortIdentificationConfiguration(); + if (!associatedCIC.Frozen) + { + yield return new ExecuteCommandAddCatalogueToCohortIdentificationSetContainer(_activator, + cohortAggregateContainer, null, null) + { SuggestedCategory = Add, OverrideCommandName = "Catalogue" }; + yield return new ExecuteCommandAddCohortSubContainer(_activator, cohortAggregateContainer) + { SuggestedCategory = Add, OverrideCommandName = "Sub Container" }; + yield return new ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer(_activator, + cohortAggregateContainer, true) + { SuggestedCategory = Add, OverrideCommandName = "Existing Cohort Set (copy of)" }; + yield return new ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer(_activator, + cohortAggregateContainer, false) + { SuggestedCategory = Add, OverrideCommandName = "Aggregate" }; + yield return new ExecuteCommandImportCohortIdentificationConfiguration(_activator, null, + cohortAggregateContainer) + { SuggestedCategory = Add, OverrideCommandName = "Existing Cohort Builder Query (copy of)" }; + + //Set Operation + yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, + SetOperation.UNION) + { SuggestedCategory = SetContainerOperation, OverrideCommandName = "UNION" }; + yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, + SetOperation.EXCEPT) + { SuggestedCategory = SetContainerOperation, OverrideCommandName = "EXCEPT" }; + yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, + SetOperation.INTERSECT) + { SuggestedCategory = SetContainerOperation, OverrideCommandName = "INTERSECT" }; + + yield return new ExecuteCommandUnMergeCohortIdentificationConfiguration(_activator, cohortAggregateContainer) - { SuggestedCategory = Add, OverrideCommandName = "Existing Cohort Builder Query (copy of)" }; - - //Set Operation - yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, - SetOperation.UNION) - { SuggestedCategory = SetContainerOperation, OverrideCommandName = "UNION" }; - yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, - SetOperation.EXCEPT) - { SuggestedCategory = SetContainerOperation, OverrideCommandName = "EXCEPT" }; - yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, - SetOperation.INTERSECT) - { SuggestedCategory = SetContainerOperation, OverrideCommandName = "INTERSECT" }; - - yield return new ExecuteCommandUnMergeCohortIdentificationConfiguration(_activator, - cohortAggregateContainer) - { OverrideCommandName = "Separate Cohort Builder Query" }; + { OverrideCommandName = "Separate Cohort Builder Query" }; + } } if (Is(o, out IDisableable disable)) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs index ac0e6265cb..eca2009d1f 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs @@ -1,6 +1,8 @@ using NPOI.OpenXmlFormats.Encryption; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.ReusableLibraryCode.Checks; using System; using System.Collections.Generic; @@ -22,11 +24,44 @@ public ExecuteCommandCreateCohortIdentificationConfigurationTemplate(IBasicActiv public override void Execute() { + //todo think about asking if we want to associate to he project if it already is + var associations = _activator.RepositoryLocator.DataExportRepository.GetAllObjects(); + var projectAssociations = associations.Where(a => a.CohortIdentificationConfiguration_ID == _cic.ID).ToList(); base.Execute(); var clone = _cic.CreateClone(ThrowImmediatelyCheckNotifier.Quiet); clone.IsTemplate = true; + clone.Freeze(); clone.Name = clone.Name.Replace("(Clone)", "") + " Template"; clone.SaveToDatabase(); + IMapsDirectlyToDatabaseTable selectedProject = null; + + if (_activator.IsInteractive && projectAssociations.Any()) { + if (projectAssociations.Count > 1) + { + //multiple, make them pick + if (_activator.YesNo("This Cohort Configuration is associated with multiple Projects. Would you like to associate this Template with one of them?", "Associate with a Project")) + { + selectedProject = _activator.SelectOne("Select a Project to associate this Template with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + } + } + else if (projectAssociations.Count == 1) + { + + //ask them if they want to use this one + if (_activator.YesNo($"This cohort configuration is already associated with the {projectAssociations.First().Project.Name} project. Would you like to associate this Template with this project?", "Use Existing Project")) + { + selectedProject = projectAssociations.First().Project; + } + } + else + { + //ask if they want to use it in a project + if (_activator.YesNo("Would you like to associate this Template with a project?", "Associate with a Project")) + { + selectedProject = _activator.SelectOne("Select a Project to associate this Template with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + } + } + } Publish(clone); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs index 68cc0d54f3..186dd82d4c 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs @@ -1,5 +1,8 @@ using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Checks; using System; using System.Collections.Generic; @@ -11,7 +14,6 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands { public class ExecuteCommandUseTemplateCohortIdentificationConfiguration : BasicCommandExecution, IAtomicCommandWithTarget { - //todo will want to think about how to associate this with a project private CohortIdentificationConfiguration _cic; private IBasicActivateItems _activator; public ExecuteCommandUseTemplateCohortIdentificationConfiguration(IBasicActivateItems activator, CohortIdentificationConfiguration cic) : base(activator) @@ -22,11 +24,49 @@ public ExecuteCommandUseTemplateCohortIdentificationConfiguration(IBasicActivate public override void Execute() { + var associations = _activator.RepositoryLocator.DataExportRepository.GetAllObjects(); + var projectAssociations = associations.Where(a => a.CohortIdentificationConfiguration_ID == _cic.ID).ToList(); + IMapsDirectlyToDatabaseTable selectedProject = null; + if (_activator.IsInteractive && projectAssociations.Any()) + { + if (projectAssociations.Count > 1) + { + //multiple, make them pick + if (_activator.YesNo("This Template is associated with multiple Projects. Would you like to associate this cohort configuration with one of them?", "Associate with a Project")) + { + selectedProject = _activator.SelectOne("Select a Project to associate this cohort with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + } + } + else if (projectAssociations.Count == 1) + { + + //ask them if they want to use this one + if (_activator.YesNo($"This template is already associated with the {projectAssociations.First().Project.Name} project. Would you like to associate this cohort configuration with this project?", "Use Existing Project")) + { + selectedProject = projectAssociations.First().Project; + } + } + else + { + //ask if they want to use it in a project + if (_activator.YesNo("Would you like to associate this cohort configuration with a project?", "Associate with a Project")) + { + selectedProject = _activator.SelectOne("Select a Project to associate this cohort with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + } + } + } base.Execute(); var clone = _cic.CreateClone(ThrowImmediatelyCheckNotifier.Quiet); clone.IsTemplate = false; clone.Name = clone.Name.Replace(" Template", ""); clone.SaveToDatabase(); + if (selectedProject != null) + { + var cmd = new ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(_activator); + cmd.SetTarget(clone); + cmd.SetTarget((Project)selectedProject); + cmd.Execute(); + } Publish(clone); } diff --git a/Rdmp.Core/DataExport/Data/Project.cs b/Rdmp.Core/DataExport/Data/Project.cs index 9855e434c3..f03690e790 100644 --- a/Rdmp.Core/DataExport/Data/Project.cs +++ b/Rdmp.Core/DataExport/Data/Project.cs @@ -176,7 +176,14 @@ public CohortIdentificationConfiguration[] GetAssociatedCohortIdentificationConf { var associations = Repository.GetAllObjectsWithParent(this); - return associations.Select(a => a.CohortIdentificationConfiguration).Where(c => c != null).ToArray(); + return associations.Select(a => a.CohortIdentificationConfiguration).Where(c => c != null && !c.IsTemplate).ToArray(); + } + + public CohortIdentificationConfiguration[] GetAssociatedTemplateCohortIdentificationConfigurations() + { + var associations = + Repository.GetAllObjectsWithParent(this); + return associations.Select(a => a.CohortIdentificationConfiguration).Where(c => c != null && c.IsTemplate).ToArray(); } /// diff --git a/Rdmp.Core/Icons/AssociatedCohortIdentificationTemplatesNode.png b/Rdmp.Core/Icons/AssociatedCohortIdentificationTemplatesNode.png new file mode 100644 index 0000000000000000000000000000000000000000..e0cdd77c8a693a318cd6a6673c2ebf7f7d67f55e GIT binary patch literal 1149 zcmV-@1cLjCP)N2bZe?^J zG%hhNF=Hy6O8@`^`$YX4YkQ3%d&}u-HOhylhFEhPFsZs@2j5 zDnF{mkQhyhCTc=7(P*Mke;BRAqP5UaZ9)^ZK8ht!Voj>4KPX~iBQ2Jvh03eI?y?=A zJG1k+*E`En+6X6ea_=|yJLk@Qd_%A@GB{H?yIdZ(;z)EM%c1MqqjnGYvLAS3 zq_;!fdjz***&HjGSw_uyD=YLJ#wN{<}q$;mVz7_m( zQ}w|41XH_TOi)|X@6p4?+Hu_&%t-psx# zQQrE@v%Q3gj4=~7Q-@uPAkmoPRMGft?Nw_{x=Fx5jJOQaU~|zLo6|25$$7`Pu<|0^ zTvBL*uIf=~^|+}9?~hUyia=mc9fqkuN=QVKKkpk!zW7^Q2GW0Sd%Ha;ceP+!<>WE5 zo~K`F^=#Z-hKfUv%d+7dO4H;FA5&8760=EmVsHDY6Mwe0;P5-e5Y;FRUY@uzI-=B* z!L|qId|TdgIFk!S@FjsU{N2-!gU?sv&Bia-ya~<20z$GqA|<&5G&i=AhT0$S)(d5b zdUvoiWuV^^?)liI%Pj&-5St?dk}wOX-C)W9^4B_1zN-i?mp(#{y;N z8%$-0_xRa>^2X%bYC=rNLf}dR>W>{kO`E6JlUue=fU(lOU4lN-3@vaP(aDogC+ZL# zuZ1z&38wL~vm>vSvXF!cGm@%_J8s%mT5BaCh@KmeD&st`DYRU)i z{7IHziOfY>MqzF!^Oj#Drp3&l$8$LZ#;b#m@+z@h?*(HHP8aXEdF18ur-BG+favEj zzdDplEW ..\AllTemplateCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - ..\TemplateCohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - + + ..\TemplateCohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AssociatedCohortIdentificationTemplatesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx index 7c96a78d8b..05da453d17 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx @@ -1,6 +1,6 @@  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - - ..\zh-Hans\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + ..\zh-Hans\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + + + + + + + \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs index cb7f3ced64..5277dbf27d 100644 --- a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs +++ b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs @@ -214,5 +214,6 @@ public enum RDMPConcept RegexRedactionConfiguration, RegexRedactionKey, AllRegexRedactionConfigurationsNode, - AllTemplateCohortIdentificationConfigurationsNode + AllTemplateCohortIdentificationConfigurationsNode, + AssociatedCohortIdentificationTemplatesNode } \ No newline at end of file diff --git a/Rdmp.Core/Providers/DataExportChildProvider.cs b/Rdmp.Core/Providers/DataExportChildProvider.cs index 8aa30541f3..267613c0d0 100644 --- a/Rdmp.Core/Providers/DataExportChildProvider.cs +++ b/Rdmp.Core/Providers/DataExportChildProvider.cs @@ -351,6 +351,23 @@ private void AddChildren(ProjectCohortsNode projectCohortsNode, DescendancyList children.Add(associatedCohortConfigurations); AddChildren(associatedCohortConfigurations, descendancy.Add(associatedCohortConfigurations)); + + var associatedTemplatesNode = new AssociatedCohortIdentificationTemplatesNode(projectCohortsNode.Project); + children.Add(associatedTemplatesNode); + AddChildren(associatedTemplatesNode, descendancy.Add(associatedTemplatesNode)); + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AssociatedCohortIdentificationTemplatesNode associatedCohortIdentificationTemplatesNode, DescendancyList descendancy) + { + var children = new HashSet(); + var associatedCohorts = associatedCohortIdentificationTemplatesNode.Project.GetAssociatedTemplateCohortIdentificationConfigurations(); + foreach (var cohort in associatedCohorts) + { + children.Add(cohort); + } + AddToDictionaries(children, descendancy); } @@ -390,6 +407,7 @@ private void AddChildren(ProjectCohortIdentificationConfigurationAssociationsNod assoc.Project_ID == projectCiCsNode.Project.ID)) { var matchingCic = AllCohortIdentificationConfigurations.SingleOrDefault(cic => + cic.ID == association.CohortIdentificationConfiguration_ID) ?? AllTemplateCohortIdentificationConfigurations.SingleOrDefault(cic => cic.ID == association.CohortIdentificationConfiguration_ID); if (matchingCic == null) diff --git a/Rdmp.Core/Providers/Nodes/CohortNodes/AssociatedCohortIdentificationTemplatesNode.cs b/Rdmp.Core/Providers/Nodes/CohortNodes/AssociatedCohortIdentificationTemplatesNode.cs new file mode 100644 index 0000000000..a9425b52da --- /dev/null +++ b/Rdmp.Core/Providers/Nodes/CohortNodes/AssociatedCohortIdentificationTemplatesNode.cs @@ -0,0 +1,40 @@ +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Providers.Nodes.CohortNodes +{ + public class AssociatedCohortIdentificationTemplatesNode:Node, IOrderable + { + public Project Project { get; set; } + + public AssociatedCohortIdentificationTemplatesNode(Project project) + { + Project = project; + } + public override string ToString() => "Associated Cohort ConfigurationTemplates"; + + protected bool Equals(AssociatedCohortIdentificationTemplatesNode other) => + Equals(Project, other.Project); + + public override bool Equals(object obj) + { + if (obj is null) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != GetType()) return false; + return Equals((AssociatedCohortIdentificationTemplatesNode)obj); + } + + public override int GetHashCode() => Project != null ? Project.GetHashCode() : 0; + + public int Order + { + get => 1; + set { } + } + } +} diff --git a/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs b/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs index 558ab5a393..dc12479c10 100644 --- a/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs @@ -142,7 +142,6 @@ public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) { Common.Activator = Activator; var descendancy = Activator.CoreChildProvider.GetDescendancyListIfAnyFor(e.Object); - //if publish event was for a child of the cic (_cic is in the objects descendancy i.e. it sits below our cic) if (descendancy != null && descendancy.Parents.Contains(Common.Configuration)) { @@ -177,7 +176,10 @@ public override void SetDatabaseObject(IActivateItems activator, CohortIdentific gbCicInfo.Text = $"Name: {databaseObject.Name}"; tbDescription.Text = $"Description: {databaseObject.Description}"; ticket.TicketText = databaseObject.Ticket; - + if (databaseObject.IsTemplate) + { + version.Visible = false; + } if (_commonFunctionality == null) { activator.RefreshBus.Subscribe(this); @@ -216,12 +218,14 @@ public override void SetDatabaseObject(IActivateItems activator, CohortIdentific CommonFunctionality.AddToMenu( new ExecuteCommandShowXmlDoc(activator, "CohortIdentificationConfiguration.QueryCachingServer_ID", "Query Caching"), "Help (What is Query Caching)"); - CommonFunctionality.Add( - new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration(activator, null).SetTarget( - databaseObject), - "Commit Cohort", - activator.CoreIconProvider.GetImage(RDMPConcept.ExtractableCohort, OverlayKind.Add)); - + if (!databaseObject.IsTemplate) + { + CommonFunctionality.Add( + new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration(activator, null).SetTarget( + databaseObject), + "Commit Cohort", + activator.CoreIconProvider.GetImage(RDMPConcept.ExtractableCohort, OverlayKind.Add)); + } foreach (var c in _timeoutControls.GetControls()) CommonFunctionality.Add(c); From 20d56804ca5d6a5307c71ce876cb0d46970b4203 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 10 Nov 2025 16:30:41 +0000 Subject: [PATCH 116/142] add db patch --- .../CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql | 11 +++++++++++ Rdmp.Core/Rdmp.Core.csproj | 1 + 2 files changed, 12 insertions(+) create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql new file mode 100644 index 0000000000..4610d3e03b --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql @@ -0,0 +1,11 @@ +--Version: 8.4.3 +--Description: Add new metadata fields for catalogues +if not exists (select 1 from sys.columns where name = 'IsTemplate' and OBJECT_NAME(object_id) = 'CohortIdentificationConfiguration') +BEGIN +ALTER TABLE [dbo].[CohortIdentificationConfiguration] +ADD +[IsTemplate] [bit] NOT NULL DEFAULT 0 +END +GO + + diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 85c918ed5f..a840864982 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -256,6 +256,7 @@ + From ecf81ad80053d87187ffd67fdfa4cca789000b9e Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 07:46:43 +0000 Subject: [PATCH 117/142] add other mapping --- Rdmp.Core/Providers/CatalogueChildProvider.cs | 3418 +++++++++-------- 1 file changed, 1710 insertions(+), 1708 deletions(-) diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 248ea43270..097aac61c1 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -54,1933 +54,1935 @@ namespace Rdmp.Core.Providers; /// public class CatalogueChildProvider : ICoreChildProvider { - //Load System - public LoadMetadata[] AllLoadMetadatas { get; set; } - public LoadMetadataCatalogueLinkage[] AllLoadMetadataCatalogueLinkages { get; set; } + //Load System + public LoadMetadata[] AllLoadMetadatas { get; set; } + public LoadMetadataCatalogueLinkage[] AllLoadMetadataCatalogueLinkages { get; set; } private LoadMetadataCatalogueLinkage[] AllLoadMetadataLinkage { get; set; } - public ProcessTask[] AllProcessTasks { get; set; } - public ProcessTaskArgument[] AllProcessTasksArguments { get; set; } + public ProcessTask[] AllProcessTasks { get; set; } + public ProcessTaskArgument[] AllProcessTasksArguments { get; set; } - public LoadProgress[] AllLoadProgresses { get; set; } - public CacheProgress[] AllCacheProgresses { get; set; } - public PermissionWindow[] AllPermissionWindows { get; set; } + public LoadProgress[] AllLoadProgresses { get; set; } + public CacheProgress[] AllCacheProgresses { get; set; } + public PermissionWindow[] AllPermissionWindows { get; set; } - //Catalogue side of things - public Catalogue[] AllCatalogues { get; set; } - public Curation.Data.Dataset[] AllDatasets { get; set; } - public Dictionary AllCataloguesDictionary { get; private set; } + //Catalogue side of things + public Catalogue[] AllCatalogues { get; set; } + public Curation.Data.Dataset[] AllDatasets { get; set; } + public Dictionary AllCataloguesDictionary { get; private set; } - public SupportingDocument[] AllSupportingDocuments { get; set; } - public SupportingSQLTable[] AllSupportingSQL { get; set; } + public SupportingDocument[] AllSupportingDocuments { get; set; } + public SupportingSQLTable[] AllSupportingSQL { get; set; } - //tells you the immediate children of a given node. Do not add to this directly instead add using AddToDictionaries unless you want the Key to be an 'on the sly' no known descendency child - private ConcurrentDictionary> _childDictionary = new(); + //tells you the immediate children of a given node. Do not add to this directly instead add using AddToDictionaries unless you want the Key to be an 'on the sly' no known descendency child + private ConcurrentDictionary> _childDictionary = new(); - //This is the reverse of _childDictionary in some ways. _childDictionary tells you the immediate children while - //this tells you for a given child object what the navigation tree down to get to it is e.g. ascendancy[child] would return [root,grandParent,parent] - private ConcurrentDictionary _descendancyDictionary = new(); + //This is the reverse of _childDictionary in some ways. _childDictionary tells you the immediate children while + //this tells you for a given child object what the navigation tree down to get to it is e.g. ascendancy[child] would return [root,grandParent,parent] + private ConcurrentDictionary _descendancyDictionary = new(); - public IEnumerable AllCatalogueItems => AllCatalogueItemsDictionary.Values; + public IEnumerable AllCatalogueItems => AllCatalogueItemsDictionary.Values; - private Dictionary> _catalogueToCatalogueItems; - public Dictionary AllCatalogueItemsDictionary { get; private set; } + private Dictionary> _catalogueToCatalogueItems; + public Dictionary AllCatalogueItemsDictionary { get; private set; } - private Dictionary _allColumnInfos; + private Dictionary _allColumnInfos; - public AggregateConfiguration[] AllAggregateConfigurations { get; private set; } - public AggregateDimension[] AllAggregateDimensions { get; private set; } + public AggregateConfiguration[] AllAggregateConfigurations { get; private set; } + public AggregateDimension[] AllAggregateDimensions { get; private set; } - public AggregateContinuousDateAxis[] AllAggregateContinuousDateAxis { get; private set; } + public AggregateContinuousDateAxis[] AllAggregateContinuousDateAxis { get; private set; } - public AllRDMPRemotesNode AllRDMPRemotesNode { get; private set; } - public RemoteRDMP[] AllRemoteRDMPs { get; set; } + public AllRDMPRemotesNode AllRDMPRemotesNode { get; private set; } + public RemoteRDMP[] AllRemoteRDMPs { get; set; } - public AllDashboardsNode AllDashboardsNode { get; set; } - public DashboardLayout[] AllDashboards { get; set; } + public AllDashboardsNode AllDashboardsNode { get; set; } + public DashboardLayout[] AllDashboards { get; set; } - public AllObjectSharingNode AllObjectSharingNode { get; private set; } - public ObjectImport[] AllImports { get; set; } - public ObjectExport[] AllExports { get; set; } + public AllObjectSharingNode AllObjectSharingNode { get; private set; } + public ObjectImport[] AllImports { get; set; } + public ObjectExport[] AllExports { get; set; } - public AllStandardRegexesNode AllStandardRegexesNode { get; private set; } - public AllPipelinesNode AllPipelinesNode { get; private set; } - public OtherPipelinesNode OtherPipelinesNode { get; private set; } - public Pipeline[] AllPipelines { get; set; } - public PipelineComponent[] AllPipelineComponents { get; set; } + public AllStandardRegexesNode AllStandardRegexesNode { get; private set; } + public AllPipelinesNode AllPipelinesNode { get; private set; } + public OtherPipelinesNode OtherPipelinesNode { get; private set; } + public Pipeline[] AllPipelines { get; set; } + public PipelineComponent[] AllPipelineComponents { get; set; } - public PipelineComponentArgument[] AllPipelineComponentsArguments { get; set; } + public PipelineComponentArgument[] AllPipelineComponentsArguments { get; set; } - public StandardRegex[] AllStandardRegexes { get; set; } + public StandardRegex[] AllStandardRegexes { get; set; } - //TableInfo side of things - public AllANOTablesNode AllANOTablesNode { get; private set; } - public ANOTable[] AllANOTables { get; set; } + //TableInfo side of things + public AllANOTablesNode AllANOTablesNode { get; private set; } + public ANOTable[] AllANOTables { get; set; } - public ExternalDatabaseServer[] AllExternalServers { get; private set; } - public TableInfoServerNode[] AllServers { get; private set; } - public TableInfo[] AllTableInfos { get; private set; } + public ExternalDatabaseServer[] AllExternalServers { get; private set; } + public TableInfoServerNode[] AllServers { get; private set; } + public TableInfo[] AllTableInfos { get; private set; } - public AllDataAccessCredentialsNode AllDataAccessCredentialsNode { get; set; } + public AllDataAccessCredentialsNode AllDataAccessCredentialsNode { get; set; } - public AllExternalServersNode AllExternalServersNode { get; private set; } - public AllServersNode AllServersNode { get; private set; } + public AllExternalServersNode AllExternalServersNode { get; private set; } + public AllServersNode AllServersNode { get; private set; } - public DataAccessCredentials[] AllDataAccessCredentials { get; set; } - public Dictionary> AllDataAccessCredentialUsages { get; set; } + public DataAccessCredentials[] AllDataAccessCredentials { get; set; } + public Dictionary> AllDataAccessCredentialUsages { get; set; } - public Dictionary> TableInfosToColumnInfos { get; private set; } - public ColumnInfo[] AllColumnInfos { get; private set; } - public PreLoadDiscardedColumn[] AllPreLoadDiscardedColumns { get; private set; } + public Dictionary> TableInfosToColumnInfos { get; private set; } + public ColumnInfo[] AllColumnInfos { get; private set; } + public PreLoadDiscardedColumn[] AllPreLoadDiscardedColumns { get; private set; } - public Lookup[] AllLookups { get; set; } + public Lookup[] AllLookups { get; set; } - public JoinInfo[] AllJoinInfos { get; set; } + public JoinInfo[] AllJoinInfos { get; set; } - public AnyTableSqlParameter[] AllAnyTableParameters; + public AnyTableSqlParameter[] AllAnyTableParameters; - //Filter / extraction side of things - public IEnumerable AllExtractionInformations => AllExtractionInformationsDictionary.Values; + //Filter / extraction side of things + public IEnumerable AllExtractionInformations => AllExtractionInformationsDictionary.Values; - public AllPermissionWindowsNode AllPermissionWindowsNode { get; set; } - public FolderNode LoadMetadataRootFolder { get; set; } + public AllPermissionWindowsNode AllPermissionWindowsNode { get; set; } + public FolderNode LoadMetadataRootFolder { get; set; } - public FolderNode DatasetRootFolder { get; set; } + public FolderNode DatasetRootFolder { get; set; } public FolderNode CohortIdentificationConfigurationRootFolder { get; set; } //public FolderNode TemplateCohortIdentificationConfigurationRootFolder { get; set; } - public AllTemplateCohortIdentificationConfigurationsNode AllTemplateCohortIdentificationConfigurationsNode { get; set; } + public AllTemplateCohortIdentificationConfigurationsNode AllTemplateCohortIdentificationConfigurationsNode { get; set; } public FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; set; } - public AllConnectionStringKeywordsNode AllConnectionStringKeywordsNode { get; set; } - public ConnectionStringKeyword[] AllConnectionStringKeywords { get; set; } + public AllConnectionStringKeywordsNode AllConnectionStringKeywordsNode { get; set; } + public ConnectionStringKeyword[] AllConnectionStringKeywords { get; set; } - public Dictionary AllExtractionInformationsDictionary { get; private set; } - protected Dictionary _extractionInformationsByCatalogueItem; + public Dictionary AllExtractionInformationsDictionary { get; private set; } + protected Dictionary _extractionInformationsByCatalogueItem; - private IFilterManager _aggregateFilterManager; + private IFilterManager _aggregateFilterManager; - //Filters for Aggregates (includes filter containers (AND/OR) - public Dictionary AllAggregateContainersDictionary { get; private set; } - public AggregateFilterContainer[] AllAggregateContainers => AllAggregateContainersDictionary.Values.ToArray(); + //Filters for Aggregates (includes filter containers (AND/OR) + public Dictionary AllAggregateContainersDictionary { get; private set; } + public AggregateFilterContainer[] AllAggregateContainers => AllAggregateContainersDictionary.Values.ToArray(); - public AggregateFilter[] AllAggregateFilters { get; private set; } - public AggregateFilterParameter[] AllAggregateFilterParameters { get; private set; } + public AggregateFilter[] AllAggregateFilters { get; private set; } + public AggregateFilterParameter[] AllAggregateFilterParameters { get; private set; } - //Catalogue master filters (does not include any support for filter containers (AND/OR) - private ExtractionFilter[] AllCatalogueFilters; - public ExtractionFilterParameter[] AllCatalogueParameters; - public ExtractionFilterParameterSet[] AllCatalogueValueSets; - public ExtractionFilterParameterSetValue[] AllCatalogueValueSetValues; + //Catalogue master filters (does not include any support for filter containers (AND/OR) + private ExtractionFilter[] AllCatalogueFilters; + public ExtractionFilterParameter[] AllCatalogueParameters; + public ExtractionFilterParameterSet[] AllCatalogueValueSets; + public ExtractionFilterParameterSetValue[] AllCatalogueValueSetValues; - private ICohortContainerManager _cohortContainerManager; + private ICohortContainerManager _cohortContainerManager; public CohortIdentificationConfiguration[] AllCohortIdentificationConfigurations { get; private set; } public CohortIdentificationConfiguration[] AllTemplateCohortIdentificationConfigurations { get; private set; } public CohortAggregateContainer[] AllCohortAggregateContainers { get; set; } - public JoinableCohortAggregateConfiguration[] AllJoinables { get; set; } - public JoinableCohortAggregateConfigurationUse[] AllJoinUses { get; set; } + public JoinableCohortAggregateConfiguration[] AllJoinables { get; set; } + public JoinableCohortAggregateConfigurationUse[] AllJoinUses { get; set; } - /// - /// Collection of all objects for which there are masqueraders - /// - public ConcurrentDictionary> AllMasqueraders { get; private set; } + /// + /// Collection of all objects for which there are masqueraders + /// + public ConcurrentDictionary> AllMasqueraders { get; private set; } - private IChildProvider[] _pluginChildProviders; - private readonly ICatalogueRepository _catalogueRepository; - private readonly ICheckNotifier _errorsCheckNotifier; - private readonly List _blockedPlugins = new(); + private IChildProvider[] _pluginChildProviders; + private readonly ICatalogueRepository _catalogueRepository; + private readonly ICheckNotifier _errorsCheckNotifier; + private readonly List _blockedPlugins = new(); - public AllGovernanceNode AllGovernanceNode { get; private set; } - public GovernancePeriod[] AllGovernancePeriods { get; private set; } - public GovernanceDocument[] AllGovernanceDocuments { get; private set; } - public Dictionary> GovernanceCoverage { get; private set; } + public AllGovernanceNode AllGovernanceNode { get; private set; } + public GovernancePeriod[] AllGovernancePeriods { get; private set; } + public GovernanceDocument[] AllGovernanceDocuments { get; private set; } + public Dictionary> GovernanceCoverage { get; private set; } - private CommentStore _commentStore; + private CommentStore _commentStore; - public JoinableCohortAggregateConfigurationUse[] AllJoinableCohortAggregateConfigurationUse { get; private set; } - public AllPluginsNode AllPluginsNode { get; private set; } - public HashSet PipelineUseCases { get; set; } = new(); + public JoinableCohortAggregateConfigurationUse[] AllJoinableCohortAggregateConfigurationUse { get; private set; } + public AllPluginsNode AllPluginsNode { get; private set; } + public HashSet PipelineUseCases { get; set; } = new(); - /// - /// Lock for changes to Child provider - /// - protected object WriteLock = new(); + /// + /// Lock for changes to Child provider + /// + protected object WriteLock = new(); - public AllOrphanAggregateConfigurationsNode OrphanAggregateConfigurationsNode { get; set; } = new(); - public AllTemplateAggregateConfigurationsNode TemplateAggregateConfigurationsNode { get; set; } = new(); - public FolderNode CatalogueRootFolder { get; private set; } + public AllOrphanAggregateConfigurationsNode OrphanAggregateConfigurationsNode { get; set; } = new(); + public AllTemplateAggregateConfigurationsNode TemplateAggregateConfigurationsNode { get; set; } = new(); + public FolderNode CatalogueRootFolder { get; private set; } - public AllDatasetsNode AllDatasetsNode { get; set; } + public AllDatasetsNode AllDatasetsNode { get; set; } - public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; set; } - public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; set; } + public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; set; } + public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; set; } - public HashSet OrphanAggregateConfigurations; - public AggregateConfiguration[] TemplateAggregateConfigurations; + public HashSet OrphanAggregateConfigurations; + public AggregateConfiguration[] TemplateAggregateConfigurations; - protected Stopwatch ProgressStopwatch = Stopwatch.StartNew(); - private int _progress; + protected Stopwatch ProgressStopwatch = Stopwatch.StartNew(); + private int _progress; - /// - /// - /// - /// - /// - /// Where to report errors building the hierarchy e.g. when crash. Set to null for - /// Previous child provider state if you know it otherwise null - public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] pluginChildProviders, - ICheckNotifier errorsCheckNotifier, CatalogueChildProvider previousStateIfKnown) - { - _commentStore = repository.CommentStore; - _catalogueRepository = repository; - _catalogueRepository?.EncryptionManager?.ClearAllInjections(); + /// + /// + /// + /// + /// + /// Where to report errors building the hierarchy e.g. when crash. Set to null for + /// Previous child provider state if you know it otherwise null + public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] pluginChildProviders, + ICheckNotifier errorsCheckNotifier, CatalogueChildProvider previousStateIfKnown) + { + _commentStore = repository.CommentStore; + _catalogueRepository = repository; + _catalogueRepository?.EncryptionManager?.ClearAllInjections(); - _errorsCheckNotifier = errorsCheckNotifier ?? IgnoreAllErrorsCheckNotifier.Instance; + _errorsCheckNotifier = errorsCheckNotifier ?? IgnoreAllErrorsCheckNotifier.Instance; - if (UserSettings.DebugPerformance) - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( - $"Refresh generated by:{Environment.NewLine}{Environment.StackTrace}", CheckResult.Success)); + if (UserSettings.DebugPerformance) + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( + $"Refresh generated by:{Environment.NewLine}{Environment.StackTrace}", CheckResult.Success)); - // all the objects which are - AllMasqueraders = new ConcurrentDictionary>(); + // all the objects which are + AllMasqueraders = new ConcurrentDictionary>(); - _pluginChildProviders = pluginChildProviders ?? Array.Empty(); + _pluginChildProviders = pluginChildProviders ?? Array.Empty(); - ReportProgress("Before object fetches"); + ReportProgress("Before object fetches"); - AllAnyTableParameters = GetAllObjects(repository); + AllAnyTableParameters = GetAllObjects(repository); - AllANOTables = GetAllObjects(repository); - AllANOTablesNode = new AllANOTablesNode(); - AddChildren(AllANOTablesNode); + AllANOTables = GetAllObjects(repository); + AllANOTablesNode = new AllANOTablesNode(); + AddChildren(AllANOTablesNode); - AllCatalogues = GetAllObjects(repository); - AllCataloguesDictionary = AllCatalogues.ToDictionaryEx(i => i.ID, o => o); + AllCatalogues = GetAllObjects(repository); + AllCataloguesDictionary = AllCatalogues.ToDictionaryEx(i => i.ID, o => o); - AllDatasets = GetAllObjects(repository); + AllDatasets = GetAllObjects(repository); - AllLoadMetadatas = GetAllObjects(repository); - AllLoadMetadataCatalogueLinkages = GetAllObjects(repository); + AllLoadMetadatas = GetAllObjects(repository); + AllLoadMetadataCatalogueLinkages = GetAllObjects(repository); AllLoadMetadataLinkage = GetAllObjects(repository); - AllProcessTasks = GetAllObjects(repository); - AllProcessTasksArguments = GetAllObjects(repository); - AllLoadProgresses = GetAllObjects(repository); - AllCacheProgresses = GetAllObjects(repository); + AllProcessTasks = GetAllObjects(repository); + AllProcessTasksArguments = GetAllObjects(repository); + AllLoadProgresses = GetAllObjects(repository); + AllCacheProgresses = GetAllObjects(repository); - AllPermissionWindows = GetAllObjects(repository); - AllPermissionWindowsNode = new AllPermissionWindowsNode(); - AddChildren(AllPermissionWindowsNode); + AllPermissionWindows = GetAllObjects(repository); + AllPermissionWindowsNode = new AllPermissionWindowsNode(); + AddChildren(AllPermissionWindowsNode); - AllRemoteRDMPs = GetAllObjects(repository); + AllRemoteRDMPs = GetAllObjects(repository); - AllExternalServers = GetAllObjects(repository); + AllExternalServers = GetAllObjects(repository); - AllTableInfos = GetAllObjects(repository); - AllDataAccessCredentials = GetAllObjects(repository); - AllDataAccessCredentialsNode = new AllDataAccessCredentialsNode(); - AddChildren(AllDataAccessCredentialsNode); + AllTableInfos = GetAllObjects(repository); + AllDataAccessCredentials = GetAllObjects(repository); + AllDataAccessCredentialsNode = new AllDataAccessCredentialsNode(); + AddChildren(AllDataAccessCredentialsNode); - AllConnectionStringKeywordsNode = new AllConnectionStringKeywordsNode(); - AllConnectionStringKeywords = GetAllObjects(repository).ToArray(); - AddToDictionaries(new HashSet(AllConnectionStringKeywords), - new DescendancyList(AllConnectionStringKeywordsNode)); + AllConnectionStringKeywordsNode = new AllConnectionStringKeywordsNode(); + AllConnectionStringKeywords = GetAllObjects(repository).ToArray(); + AddToDictionaries(new HashSet(AllConnectionStringKeywords), + new DescendancyList(AllConnectionStringKeywordsNode)); - ReportProgress("after basic object fetches"); + ReportProgress("after basic object fetches"); - Task.WaitAll( - //which TableInfos use which Credentials under which DataAccessContexts - Task.Factory.StartNew(() => - { - AllDataAccessCredentialUsages = - repository.TableInfoCredentialsManager.GetAllCredentialUsagesBy(AllDataAccessCredentials, - AllTableInfos); - }), - Task.Factory.StartNew(() => { AllColumnInfos = GetAllObjects(repository); }) - ); + Task.WaitAll( + //which TableInfos use which Credentials under which DataAccessContexts + Task.Factory.StartNew(() => + { + AllDataAccessCredentialUsages = + repository.TableInfoCredentialsManager.GetAllCredentialUsagesBy(AllDataAccessCredentials, + AllTableInfos); + }), + Task.Factory.StartNew(() => { AllColumnInfos = GetAllObjects(repository); }) + ); - ReportProgress("After credentials"); + ReportProgress("After credentials"); - TableInfosToColumnInfos = AllColumnInfos.GroupBy(c => c.TableInfo_ID) - .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); + TableInfosToColumnInfos = AllColumnInfos.GroupBy(c => c.TableInfo_ID) + .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); - ReportProgress("After TableInfo to ColumnInfo mapping"); + ReportProgress("After TableInfo to ColumnInfo mapping"); - AllPreLoadDiscardedColumns = GetAllObjects(repository); + AllPreLoadDiscardedColumns = GetAllObjects(repository); - AllSupportingDocuments = GetAllObjects(repository); - AllSupportingSQL = GetAllObjects(repository); + AllSupportingDocuments = GetAllObjects(repository); + AllSupportingSQL = GetAllObjects(repository); - AllCohortIdentificationConfigurations = GetAllObjects(repository).Where(cic => !cic.IsTemplate).ToArray(); + AllCohortIdentificationConfigurations = GetAllObjects(repository).Where(cic => !cic.IsTemplate).ToArray(); AllTemplateCohortIdentificationConfigurations = GetAllObjects(repository).Where(cic => cic.IsTemplate).ToArray(); FetchCatalogueItems(); - ReportProgress("After CatalogueItem injection"); + ReportProgress("After CatalogueItem injection"); - FetchExtractionInformations(); + FetchExtractionInformations(); - ReportProgress("After ExtractionInformation injection"); + ReportProgress("After ExtractionInformation injection"); - BuildAggregateConfigurations(); + BuildAggregateConfigurations(); - BuildCohortCohortAggregateContainers(); + BuildCohortCohortAggregateContainers(); - AllJoinables = GetAllObjects(repository); - AllJoinUses = GetAllObjects(repository); + AllJoinables = GetAllObjects(repository); + AllJoinUses = GetAllObjects(repository); - AllCatalogueFilters = GetAllObjects(repository); - AllCatalogueParameters = GetAllObjects(repository); - AllCatalogueValueSets = GetAllObjects(repository); - AllCatalogueValueSetValues = GetAllObjects(repository); + AllCatalogueFilters = GetAllObjects(repository); + AllCatalogueParameters = GetAllObjects(repository); + AllCatalogueValueSets = GetAllObjects(repository); + AllCatalogueValueSetValues = GetAllObjects(repository); - ReportProgress("After Filter and Joinable fetching"); + ReportProgress("After Filter and Joinable fetching"); - AllLookups = GetAllObjects(repository); + AllLookups = GetAllObjects(repository); - foreach (var l in AllLookups) - l.SetKnownColumns(_allColumnInfos[l.PrimaryKey_ID], _allColumnInfos[l.ForeignKey_ID], - _allColumnInfos[l.Description_ID]); + foreach (var l in AllLookups) + l.SetKnownColumns(_allColumnInfos[l.PrimaryKey_ID], _allColumnInfos[l.ForeignKey_ID], + _allColumnInfos[l.Description_ID]); - AllJoinInfos = repository.GetAllObjects(); + AllJoinInfos = repository.GetAllObjects(); - foreach (var j in AllJoinInfos) - j.SetKnownColumns(_allColumnInfos[j.PrimaryKey_ID], _allColumnInfos[j.ForeignKey_ID]); + foreach (var j in AllJoinInfos) + j.SetKnownColumns(_allColumnInfos[j.PrimaryKey_ID], _allColumnInfos[j.ForeignKey_ID]); - ReportProgress("After SetKnownColumns"); + ReportProgress("After SetKnownColumns"); - AllExternalServersNode = new AllExternalServersNode(); - AddChildren(AllExternalServersNode); + AllExternalServersNode = new AllExternalServersNode(); + AddChildren(AllExternalServersNode); - AllRDMPRemotesNode = new AllRDMPRemotesNode(); - AddChildren(AllRDMPRemotesNode); + AllRDMPRemotesNode = new AllRDMPRemotesNode(); + AddChildren(AllRDMPRemotesNode); - AllDashboardsNode = new AllDashboardsNode(); - AllDashboards = GetAllObjects(repository); - AddChildren(AllDashboardsNode); + AllDashboardsNode = new AllDashboardsNode(); + AllDashboards = GetAllObjects(repository); + AddChildren(AllDashboardsNode); - AllObjectSharingNode = new AllObjectSharingNode(); - AllExports = GetAllObjects(repository); - AllImports = GetAllObjects(repository); + AllObjectSharingNode = new AllObjectSharingNode(); + AllExports = GetAllObjects(repository); + AllImports = GetAllObjects(repository); - AddChildren(AllObjectSharingNode); + AddChildren(AllObjectSharingNode); - ReportProgress("After Object Sharing discovery"); + ReportProgress("After Object Sharing discovery"); - //Pipelines setup (see also DataExportChildProvider for calls to AddPipelineUseCases) - //Root node for all pipelines - AllPipelinesNode = new AllPipelinesNode(); + //Pipelines setup (see also DataExportChildProvider for calls to AddPipelineUseCases) + //Root node for all pipelines + AllPipelinesNode = new AllPipelinesNode(); - //Pipelines not found to be part of any use case after AddPipelineUseCases - OtherPipelinesNode = new OtherPipelinesNode(); - AllPipelines = GetAllObjects(repository); - AllPipelineComponents = GetAllObjects(repository); - AllPipelineComponentsArguments = GetAllObjects(repository); + //Pipelines not found to be part of any use case after AddPipelineUseCases + OtherPipelinesNode = new OtherPipelinesNode(); + AllPipelines = GetAllObjects(repository); + AllPipelineComponents = GetAllObjects(repository); + AllPipelineComponentsArguments = GetAllObjects(repository); - foreach (var p in AllPipelines) - p.InjectKnown(AllPipelineComponents.Where(pc => pc.Pipeline_ID == p.ID).ToArray()); + foreach (var p in AllPipelines) + p.InjectKnown(AllPipelineComponents.Where(pc => pc.Pipeline_ID == p.ID).ToArray()); - AllStandardRegexesNode = new AllStandardRegexesNode(); - AllStandardRegexes = GetAllObjects(repository); - AddToDictionaries(new HashSet(AllStandardRegexes), new DescendancyList(AllStandardRegexesNode)); + AllStandardRegexesNode = new AllStandardRegexesNode(); + AllStandardRegexes = GetAllObjects(repository); + AddToDictionaries(new HashSet(AllStandardRegexes), new DescendancyList(AllStandardRegexesNode)); - ReportProgress("After Pipelines setup"); + ReportProgress("After Pipelines setup"); - //All the things for TableInfoCollectionUI - BuildServerNodes(); + //All the things for TableInfoCollectionUI + BuildServerNodes(); - ReportProgress("BuildServerNodes"); + ReportProgress("BuildServerNodes"); - //add a new CatalogueItemNodes - InjectCatalogueItems(); + //add a new CatalogueItemNodes + InjectCatalogueItems(); - CatalogueRootFolder = FolderHelper.BuildFolderTree(AllCatalogues); - AddChildren(CatalogueRootFolder, new DescendancyList(CatalogueRootFolder)); + CatalogueRootFolder = FolderHelper.BuildFolderTree(AllCatalogues); + AddChildren(CatalogueRootFolder, new DescendancyList(CatalogueRootFolder)); - DatasetRootFolder = FolderHelper.BuildFolderTree(AllDatasets); - AddChildren(DatasetRootFolder, new DescendancyList(DatasetRootFolder)); + DatasetRootFolder = FolderHelper.BuildFolderTree(AllDatasets); + AddChildren(DatasetRootFolder, new DescendancyList(DatasetRootFolder)); - ReportProgress("Build Catalogue Folder Root"); + ReportProgress("Build Catalogue Folder Root"); - LoadMetadataRootFolder = FolderHelper.BuildFolderTree(AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID is null).ToArray()); - AddChildren(LoadMetadataRootFolder, new DescendancyList(LoadMetadataRootFolder)); + LoadMetadataRootFolder = FolderHelper.BuildFolderTree(AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID is null).ToArray()); + AddChildren(LoadMetadataRootFolder, new DescendancyList(LoadMetadataRootFolder)); - CohortIdentificationConfigurationRootFolder = - FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations); - AddChildren(CohortIdentificationConfigurationRootFolder, - new DescendancyList(CohortIdentificationConfigurationRootFolder)); + CohortIdentificationConfigurationRootFolder = + FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations); + AddChildren(CohortIdentificationConfigurationRootFolder, + new DescendancyList(CohortIdentificationConfigurationRootFolder)); - CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations = FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations.Where(cic => cic.Version is null).ToArray()); - AddChildren(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations, - new DescendancyList(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations)); - - var templateAggregateConfigurationIds = - new HashSet( - repository.GetExtendedProperties(ExtendedProperty.IsTemplate) - .Where(p => p.ReferencedObjectType.Equals(nameof(AggregateConfiguration))) - .Select(r => r.ReferencedObjectID)); + CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations = FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations.Where(cic => cic.Version is null).ToArray()); + AddChildren(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations, + new DescendancyList(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations)); + + var templateAggregateConfigurationIds = + new HashSet( + repository.GetExtendedProperties(ExtendedProperty.IsTemplate) + .Where(p => p.ReferencedObjectType.Equals(nameof(AggregateConfiguration))) + .Select(r => r.ReferencedObjectID)); //TemplateCohortIdentificationConfigurationRootFolder = FolderHelper.BuildFolderTree(AllTemplateCohortIdentificationConfigurations); //AddChildren(TemplateCohortIdentificationConfigurationRootFolder, // new DescendancyList(TemplateCohortIdentificationConfigurationRootFolder)); AllTemplateCohortIdentificationConfigurationsNode = new AllTemplateCohortIdentificationConfigurationsNode(); - var templateCICTree = FolderHelper.BuildFolderTree(AllTemplateCohortIdentificationConfigurations); + var templateCICTree = FolderHelper.BuildFolderTree(AllTemplateCohortIdentificationConfigurations); AddChildren(templateCICTree, new DescendancyList(AllTemplateCohortIdentificationConfigurationsNode)); TemplateAggregateConfigurations = AllAggregateConfigurations - .Where(ac => templateAggregateConfigurationIds.Contains(ac.ID)).ToArray(); + .Where(ac => templateAggregateConfigurationIds.Contains(ac.ID)).ToArray(); - //add the orphans under the orphan folder - AddToDictionaries(new HashSet(OrphanAggregateConfigurations), - new DescendancyList(OrphanAggregateConfigurationsNode)); + //add the orphans under the orphan folder + AddToDictionaries(new HashSet(OrphanAggregateConfigurations), + new DescendancyList(OrphanAggregateConfigurationsNode)); - var dec = new DescendancyList(TemplateAggregateConfigurationsNode); - dec.SetBetterRouteExists(); - AddToDictionaries(new HashSet(TemplateAggregateConfigurations), dec); + var dec = new DescendancyList(TemplateAggregateConfigurationsNode); + dec.SetBetterRouteExists(); + AddToDictionaries(new HashSet(TemplateAggregateConfigurations), dec); - //Some AggregateConfigurations are 'Patient Index Tables', this happens when there is an existing JoinableCohortAggregateConfiguration declared where - //the AggregateConfiguration_ID is the AggregateConfiguration.ID. We can inject this knowledge now so to avoid database lookups later (e.g. at icon provision time) - var joinableDictionaryByAggregateConfigurationId = - AllJoinables.ToDictionaryEx(j => j.AggregateConfiguration_ID, v => v); + //Some AggregateConfigurations are 'Patient Index Tables', this happens when there is an existing JoinableCohortAggregateConfiguration declared where + //the AggregateConfiguration_ID is the AggregateConfiguration.ID. We can inject this knowledge now so to avoid database lookups later (e.g. at icon provision time) + var joinableDictionaryByAggregateConfigurationId = + AllJoinables.ToDictionaryEx(j => j.AggregateConfiguration_ID, v => v); - foreach (var ac in AllAggregateConfigurations) //if there's a joinable - ac.InjectKnown( //inject that we know the joinable (and what it is) - joinableDictionaryByAggregateConfigurationId.GetValueOrDefault(ac.ID)); //otherwise inject that it is not a joinable (suppresses database checking later) + foreach (var ac in AllAggregateConfigurations) //if there's a joinable + ac.InjectKnown( //inject that we know the joinable (and what it is) + joinableDictionaryByAggregateConfigurationId.GetValueOrDefault(ac.ID)); //otherwise inject that it is not a joinable (suppresses database checking later) - ReportProgress("After AggregateConfiguration injection"); + ReportProgress("After AggregateConfiguration injection"); - AllGovernanceNode = new AllGovernanceNode(); - AllGovernancePeriods = GetAllObjects(repository); - AllGovernanceDocuments = GetAllObjects(repository); - GovernanceCoverage = repository.GovernanceManager.GetAllGovernedCataloguesForAllGovernancePeriods(); + AllGovernanceNode = new AllGovernanceNode(); + AllGovernancePeriods = GetAllObjects(repository); + AllGovernanceDocuments = GetAllObjects(repository); + GovernanceCoverage = repository.GovernanceManager.GetAllGovernedCataloguesForAllGovernancePeriods(); - AddChildren(AllGovernanceNode); + AddChildren(AllGovernanceNode); - ReportProgress("After Governance"); + ReportProgress("After Governance"); - AllPluginsNode = new AllPluginsNode(); - AddChildren(AllPluginsNode); + AllPluginsNode = new AllPluginsNode(); + AddChildren(AllPluginsNode); - ReportProgress("After Plugins"); + ReportProgress("After Plugins"); - AllRegexRedactionConfigurations = GetAllObjects(repository); - AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); - AddChildren(AllRegexRedactionConfigurationsNode); + AllRegexRedactionConfigurations = GetAllObjects(repository); + AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); + AddChildren(AllRegexRedactionConfigurationsNode); - AllDatasets = GetAllObjects(repository); - AllDatasetsNode = new AllDatasetsNode(); - AddChildren(AllDatasetsNode); + AllDatasets = GetAllObjects(repository); + AllDatasetsNode = new AllDatasetsNode(); + AddChildren(AllDatasetsNode); - ReportProgress("After Configurations"); + ReportProgress("After Configurations"); - var searchables = new Dictionary>(); + var searchables = new Dictionary>(); - foreach (var o in _descendancyDictionary.Keys.OfType()) - { - if (!searchables.ContainsKey(o.ID)) - searchables.Add(o.ID, new HashSet()); + foreach (var o in _descendancyDictionary.Keys.OfType()) + { + if (!searchables.ContainsKey(o.ID)) + searchables.Add(o.ID, new HashSet()); - searchables[o.ID].Add(o); - } + searchables[o.ID].Add(o); + } - ReportProgress("After building Searchables"); + ReportProgress("After building Searchables"); - foreach (var e in AllExports) - { - if (!searchables.TryGetValue(e.ReferencedObjectID, out var searchable)) - continue; + foreach (var e in AllExports) + { + if (!searchables.TryGetValue(e.ReferencedObjectID, out var searchable)) + continue; - var known = searchable - .FirstOrDefault(s => e.ReferencedObjectType == s.GetType().FullName); + var known = searchable + .FirstOrDefault(s => e.ReferencedObjectType == s.GetType().FullName); - if (known != null) - e.InjectKnown(known); - } + if (known != null) + e.InjectKnown(known); + } - ReportProgress("After building exports"); - } + ReportProgress("After building exports"); + } - private void FetchCatalogueItems() - { - AllCatalogueItemsDictionary = - GetAllObjects(_catalogueRepository).ToDictionaryEx(i => i.ID, o => o); + private void FetchCatalogueItems() + { + AllCatalogueItemsDictionary = + GetAllObjects(_catalogueRepository).ToDictionaryEx(i => i.ID, o => o); - ReportProgress("After CatalogueItem getting"); + ReportProgress("After CatalogueItem getting"); - _catalogueToCatalogueItems = AllCatalogueItems.GroupBy(c => c.Catalogue_ID) - .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); - _allColumnInfos = AllColumnInfos.ToDictionaryEx(i => i.ID, o => o); + _catalogueToCatalogueItems = AllCatalogueItems.GroupBy(c => c.Catalogue_ID) + .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); + _allColumnInfos = AllColumnInfos.ToDictionaryEx(i => i.ID, o => o); - ReportProgress("After CatalogueItem Dictionary building"); + ReportProgress("After CatalogueItem Dictionary building"); - //Inject known ColumnInfos into CatalogueItems - Parallel.ForEach(AllCatalogueItems, ci => - { - if (ci.ColumnInfo_ID != null && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) - ci.InjectKnown(col); - else - ci.InjectKnown((ColumnInfo)null); - }); - } - - private void FetchExtractionInformations() - { - AllExtractionInformationsDictionary = GetAllObjects(_catalogueRepository) - .ToDictionaryEx(i => i.ID, o => o); - _extractionInformationsByCatalogueItem = - AllExtractionInformationsDictionary.Values.ToDictionaryEx(k => k.CatalogueItem_ID, v => v); - - //Inject known CatalogueItems into ExtractionInformations - foreach (var ei in AllExtractionInformationsDictionary.Values) - if (AllCatalogueItemsDictionary.TryGetValue(ei.CatalogueItem_ID, out var ci)) - { - ei.InjectKnown(ci.ColumnInfo); - ei.InjectKnown(ci); - } - } - - private void BuildCohortCohortAggregateContainers() - { - AllCohortAggregateContainers = GetAllObjects(_catalogueRepository); - - - //if we have a database repository then we should get answers from the caching version CohortContainerManagerFromChildProvider otherwise - //just use the one that is configured on the repository. - - _cohortContainerManager = _catalogueRepository is CatalogueRepository cataRepo - ? new CohortContainerManagerFromChildProvider(cataRepo, this) - : _catalogueRepository.CohortContainerManager; - } - - private void BuildAggregateConfigurations() - { - AllJoinableCohortAggregateConfigurationUse = - GetAllObjects(_catalogueRepository); - AllAggregateConfigurations = GetAllObjects(_catalogueRepository); - - BuildAggregateDimensions(); - - //to start with all aggregates are orphans (we prune this as we determine descendency in AddChildren methods - OrphanAggregateConfigurations = - new HashSet( - AllAggregateConfigurations.Where(ac => ac.IsCohortIdentificationAggregate)); - - foreach (var configuration in AllAggregateConfigurations) - { - configuration.InjectKnown(AllCataloguesDictionary[configuration.Catalogue_ID]); - configuration.InjectKnown(AllAggregateDimensions.Where(d => d.AggregateConfiguration_ID == configuration.ID) - .ToArray()); - } - - foreach (var d in AllAggregateDimensions) - d.InjectKnown(AllExtractionInformationsDictionary[d.ExtractionInformation_ID]); - - ReportProgress("AggregateDimension injections"); - - BuildAggregateFilterContainers(); - } - - private void BuildAggregateDimensions() - { - AllAggregateDimensions = GetAllObjects(_catalogueRepository); - AllAggregateContinuousDateAxis = GetAllObjects(_catalogueRepository); - } - - private void BuildAggregateFilterContainers() - { - AllAggregateContainersDictionary = GetAllObjects(_catalogueRepository) - .ToDictionaryEx(o => o.ID, o2 => o2); - AllAggregateFilters = GetAllObjects(_catalogueRepository); - AllAggregateFilterParameters = GetAllObjects(_catalogueRepository); - - _aggregateFilterManager = _catalogueRepository is CatalogueRepository cataRepo - ? new FilterManagerFromChildProvider(cataRepo, this) - : _catalogueRepository.FilterManager; - } - - - protected void ReportProgress(string desc) - { - if (UserSettings.DebugPerformance) - { - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( - $"ChildProvider Stage {_progress++} ({desc}):{ProgressStopwatch.ElapsedMilliseconds}ms", - CheckResult.Success)); - ProgressStopwatch.Restart(); - } - } - - private void AddChildren(AllPluginsNode allPluginsNode) - { - var children = new HashSet(LoadModuleAssembly.Assemblies); - var descendancy = new DescendancyList(allPluginsNode); - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) - { - var children = new HashSet(AllRegexRedactionConfigurations); - var descendancy = new DescendancyList(allRegexRedactionConfigurationsNode); - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllDatasetsNode allDatasetsNode) - { - var children = new HashSet(AllDatasets); - var descendancy = new DescendancyList(allDatasetsNode); - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllGovernanceNode allGovernanceNode) - { - var children = new HashSet(); - var descendancy = new DescendancyList(allGovernanceNode); - - foreach (var gp in AllGovernancePeriods) - { - children.Add(gp); - AddChildren(gp, descendancy.Add(gp)); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(GovernancePeriod governancePeriod, DescendancyList descendancy) - { - var children = new HashSet(); - - foreach (var doc in AllGovernanceDocuments.Where(d => d.GovernancePeriod_ID == governancePeriod.ID)) - children.Add(doc); - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllPermissionWindowsNode allPermissionWindowsNode) - { - var descendancy = new DescendancyList(allPermissionWindowsNode); - - foreach (var permissionWindow in AllPermissionWindows) - AddChildren(permissionWindow, descendancy.Add(permissionWindow)); - - - AddToDictionaries(new HashSet(AllPermissionWindows), descendancy); - } - - private void AddChildren(PermissionWindow permissionWindow, DescendancyList descendancy) - { - var children = new HashSet(); - - foreach (var cacheProgress in AllCacheProgresses) - if (cacheProgress.PermissionWindow_ID == permissionWindow.ID) - children.Add(new PermissionWindowUsedByCacheProgressNode(cacheProgress, permissionWindow, false)); + //Inject known ColumnInfos into CatalogueItems + Parallel.ForEach(AllCatalogueItems, ci => + { + if (ci.ColumnInfo_ID != null && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) + ci.InjectKnown(col); + else + ci.InjectKnown((ColumnInfo)null); + }); + } + + private void FetchExtractionInformations() + { + AllExtractionInformationsDictionary = GetAllObjects(_catalogueRepository) + .ToDictionaryEx(i => i.ID, o => o); + _extractionInformationsByCatalogueItem = + AllExtractionInformationsDictionary.Values.ToDictionaryEx(k => k.CatalogueItem_ID, v => v); + + //Inject known CatalogueItems into ExtractionInformations + foreach (var ei in AllExtractionInformationsDictionary.Values) + if (AllCatalogueItemsDictionary.TryGetValue(ei.CatalogueItem_ID, out var ci)) + { + ei.InjectKnown(ci.ColumnInfo); + ei.InjectKnown(ci); + } + } + + private void BuildCohortCohortAggregateContainers() + { + AllCohortAggregateContainers = GetAllObjects(_catalogueRepository); + + + //if we have a database repository then we should get answers from the caching version CohortContainerManagerFromChildProvider otherwise + //just use the one that is configured on the repository. + + _cohortContainerManager = _catalogueRepository is CatalogueRepository cataRepo + ? new CohortContainerManagerFromChildProvider(cataRepo, this) + : _catalogueRepository.CohortContainerManager; + } + + private void BuildAggregateConfigurations() + { + AllJoinableCohortAggregateConfigurationUse = + GetAllObjects(_catalogueRepository); + AllAggregateConfigurations = GetAllObjects(_catalogueRepository); + + BuildAggregateDimensions(); + + //to start with all aggregates are orphans (we prune this as we determine descendency in AddChildren methods + OrphanAggregateConfigurations = + new HashSet( + AllAggregateConfigurations.Where(ac => ac.IsCohortIdentificationAggregate)); + + foreach (var configuration in AllAggregateConfigurations) + { + configuration.InjectKnown(AllCataloguesDictionary[configuration.Catalogue_ID]); + configuration.InjectKnown(AllAggregateDimensions.Where(d => d.AggregateConfiguration_ID == configuration.ID) + .ToArray()); + } + + foreach (var d in AllAggregateDimensions) + d.InjectKnown(AllExtractionInformationsDictionary[d.ExtractionInformation_ID]); + + ReportProgress("AggregateDimension injections"); + + BuildAggregateFilterContainers(); + } + + private void BuildAggregateDimensions() + { + AllAggregateDimensions = GetAllObjects(_catalogueRepository); + AllAggregateContinuousDateAxis = GetAllObjects(_catalogueRepository); + } + + private void BuildAggregateFilterContainers() + { + AllAggregateContainersDictionary = GetAllObjects(_catalogueRepository) + .ToDictionaryEx(o => o.ID, o2 => o2); + AllAggregateFilters = GetAllObjects(_catalogueRepository); + AllAggregateFilterParameters = GetAllObjects(_catalogueRepository); + + _aggregateFilterManager = _catalogueRepository is CatalogueRepository cataRepo + ? new FilterManagerFromChildProvider(cataRepo, this) + : _catalogueRepository.FilterManager; + } + + + protected void ReportProgress(string desc) + { + if (UserSettings.DebugPerformance) + { + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( + $"ChildProvider Stage {_progress++} ({desc}):{ProgressStopwatch.ElapsedMilliseconds}ms", + CheckResult.Success)); + ProgressStopwatch.Restart(); + } + } + + private void AddChildren(AllPluginsNode allPluginsNode) + { + var children = new HashSet(LoadModuleAssembly.Assemblies); + var descendancy = new DescendancyList(allPluginsNode); + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) + { + var children = new HashSet(AllRegexRedactionConfigurations); + var descendancy = new DescendancyList(allRegexRedactionConfigurationsNode); + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllDatasetsNode allDatasetsNode) + { + var children = new HashSet(AllDatasets); + var descendancy = new DescendancyList(allDatasetsNode); + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllGovernanceNode allGovernanceNode) + { + var children = new HashSet(); + var descendancy = new DescendancyList(allGovernanceNode); + + foreach (var gp in AllGovernancePeriods) + { + children.Add(gp); + AddChildren(gp, descendancy.Add(gp)); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(GovernancePeriod governancePeriod, DescendancyList descendancy) + { + var children = new HashSet(); + + foreach (var doc in AllGovernanceDocuments.Where(d => d.GovernancePeriod_ID == governancePeriod.ID)) + children.Add(doc); + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllPermissionWindowsNode allPermissionWindowsNode) + { + var descendancy = new DescendancyList(allPermissionWindowsNode); + + foreach (var permissionWindow in AllPermissionWindows) + AddChildren(permissionWindow, descendancy.Add(permissionWindow)); + + + AddToDictionaries(new HashSet(AllPermissionWindows), descendancy); + } + + private void AddChildren(PermissionWindow permissionWindow, DescendancyList descendancy) + { + var children = new HashSet(); + + foreach (var cacheProgress in AllCacheProgresses) + if (cacheProgress.PermissionWindow_ID == permissionWindow.ID) + children.Add(new PermissionWindowUsedByCacheProgressNode(cacheProgress, permissionWindow, false)); - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void AddChildren(AllExternalServersNode allExternalServersNode) - { - AddToDictionaries(new HashSet(AllExternalServers), new DescendancyList(allExternalServersNode)); - } - - private void AddChildren(AllRDMPRemotesNode allRDMPRemotesNode) - { - AddToDictionaries(new HashSet(AllRemoteRDMPs), new DescendancyList(allRDMPRemotesNode)); - } + private void AddChildren(AllExternalServersNode allExternalServersNode) + { + AddToDictionaries(new HashSet(AllExternalServers), new DescendancyList(allExternalServersNode)); + } + + private void AddChildren(AllRDMPRemotesNode allRDMPRemotesNode) + { + AddToDictionaries(new HashSet(AllRemoteRDMPs), new DescendancyList(allRDMPRemotesNode)); + } - private void AddChildren(AllDashboardsNode allDashboardsNode) - { - AddToDictionaries(new HashSet(AllDashboards), new DescendancyList(allDashboardsNode)); - } + private void AddChildren(AllDashboardsNode allDashboardsNode) + { + AddToDictionaries(new HashSet(AllDashboards), new DescendancyList(allDashboardsNode)); + } - private void AddChildren(AllObjectSharingNode allObjectSharingNode) - { - var descendancy = new DescendancyList(allObjectSharingNode); + private void AddChildren(AllObjectSharingNode allObjectSharingNode) + { + var descendancy = new DescendancyList(allObjectSharingNode); - var allExportsNode = new AllObjectExportsNode(); - var allImportsNode = new AllObjectImportsNode(); + var allExportsNode = new AllObjectExportsNode(); + var allImportsNode = new AllObjectImportsNode(); - AddToDictionaries(new HashSet(AllExports), descendancy.Add(allExportsNode)); - AddToDictionaries(new HashSet(AllImports), descendancy.Add(allImportsNode)); + AddToDictionaries(new HashSet(AllExports), descendancy.Add(allExportsNode)); + AddToDictionaries(new HashSet(AllImports), descendancy.Add(allImportsNode)); - AddToDictionaries(new HashSet(new object[] { allExportsNode, allImportsNode }), descendancy); - } + AddToDictionaries(new HashSet(new object[] { allExportsNode, allImportsNode }), descendancy); + } - /// - /// Creates new s and fills it with all compatible Pipelines - do not call this method more than once - /// - protected void AddPipelineUseCases(Dictionary useCases) - { - var descendancy = new DescendancyList(AllPipelinesNode); - var children = new HashSet(); + /// + /// Creates new s and fills it with all compatible Pipelines - do not call this method more than once + /// + protected void AddPipelineUseCases(Dictionary useCases) + { + var descendancy = new DescendancyList(AllPipelinesNode); + var children = new HashSet(); - //pipelines not found to be part of any StandardPipelineUseCase - var unknownPipelines = new HashSet(AllPipelines); + //pipelines not found to be part of any StandardPipelineUseCase + var unknownPipelines = new HashSet(AllPipelines); - foreach (var useCase in useCases) - { - var node = new StandardPipelineUseCaseNode(useCase.Key, useCase.Value, _commentStore); + foreach (var useCase in useCases) + { + var node = new StandardPipelineUseCaseNode(useCase.Key, useCase.Value, _commentStore); - //keep track of all the use cases - PipelineUseCases.Add(node); + //keep track of all the use cases + PipelineUseCases.Add(node); - foreach (var pipeline in AddChildren(node, descendancy.Add(node))) - unknownPipelines.Remove(pipeline); + foreach (var pipeline in AddChildren(node, descendancy.Add(node))) + unknownPipelines.Remove(pipeline); - children.Add(node); - } + children.Add(node); + } - children.Add(OtherPipelinesNode); - OtherPipelinesNode.Pipelines.AddRange(unknownPipelines.Cast()); - AddToDictionaries(unknownPipelines, descendancy.Add(OtherPipelinesNode)); + children.Add(OtherPipelinesNode); + OtherPipelinesNode.Pipelines.AddRange(unknownPipelines.Cast()); + AddToDictionaries(unknownPipelines, descendancy.Add(OtherPipelinesNode)); - //it is the first standard use case - AddToDictionaries(children, descendancy); - } + //it is the first standard use case + AddToDictionaries(children, descendancy); + } - private IEnumerable AddChildren(StandardPipelineUseCaseNode node, DescendancyList descendancy) - { - var children = new HashSet(); + private IEnumerable AddChildren(StandardPipelineUseCaseNode node, DescendancyList descendancy) + { + var children = new HashSet(); - var repo = new MemoryRepository(); + var repo = new MemoryRepository(); - //Could be an issue here if a pipeline becomes compatible with multiple use cases. - //Should be impossible currently but one day it could be an issue especially if we were to - //support plugin use cases in this hierarchy + //Could be an issue here if a pipeline becomes compatible with multiple use cases. + //Should be impossible currently but one day it could be an issue especially if we were to + //support plugin use cases in this hierarchy - //find compatible pipelines useCase.Value - foreach (var compatiblePipeline in AllPipelines.Where(node.UseCase.GetContext().IsAllowable)) - { - var useCaseNode = new PipelineCompatibleWithUseCaseNode(repo, compatiblePipeline, node.UseCase); + //find compatible pipelines useCase.Value + foreach (var compatiblePipeline in AllPipelines.Where(node.UseCase.GetContext().IsAllowable)) + { + var useCaseNode = new PipelineCompatibleWithUseCaseNode(repo, compatiblePipeline, node.UseCase); - AddChildren(useCaseNode, descendancy.Add(useCaseNode)); + AddChildren(useCaseNode, descendancy.Add(useCaseNode)); - node.Pipelines.Add(compatiblePipeline); - children.Add(useCaseNode); - } + node.Pipelines.Add(compatiblePipeline); + children.Add(useCaseNode); + } - //it is the first standard use case - AddToDictionaries(children, descendancy); + //it is the first standard use case + AddToDictionaries(children, descendancy); - return children.Cast().Select(u => u.Pipeline); - } + return children.Cast().Select(u => u.Pipeline); + } - private void AddChildren(PipelineCompatibleWithUseCaseNode pipelineNode, DescendancyList descendancy) - { - var components = AllPipelineComponents.Where(c => c.Pipeline_ID == pipelineNode.Pipeline.ID) - .OrderBy(o => o.Order) - .ToArray(); + private void AddChildren(PipelineCompatibleWithUseCaseNode pipelineNode, DescendancyList descendancy) + { + var components = AllPipelineComponents.Where(c => c.Pipeline_ID == pipelineNode.Pipeline.ID) + .OrderBy(o => o.Order) + .ToArray(); - foreach (var component in components) - AddChildren(component, descendancy.Add(component)); + foreach (var component in components) + AddChildren(component, descendancy.Add(component)); - var children = new HashSet(components); + var children = new HashSet(components); - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void AddChildren(PipelineComponent pipelineComponent, DescendancyList descendancy) - { - var components = AllPipelineComponentsArguments.Where(c => c.PipelineComponent_ID == pipelineComponent.ID) - .ToArray(); + private void AddChildren(PipelineComponent pipelineComponent, DescendancyList descendancy) + { + var components = AllPipelineComponentsArguments.Where(c => c.PipelineComponent_ID == pipelineComponent.ID) + .ToArray(); - var children = new HashSet(components); + var children = new HashSet(components); - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void BuildServerNodes() - { - //add a root node for all the servers to be children of - AllServersNode = new AllServersNode(); + private void BuildServerNodes() + { + //add a root node for all the servers to be children of + AllServersNode = new AllServersNode(); - var descendancy = new DescendancyList(AllServersNode); - var allServers = new List(); + var descendancy = new DescendancyList(AllServersNode); + var allServers = new List(); - foreach (var typeGroup in AllTableInfos.GroupBy(t => t.DatabaseType)) - { - var dbType = typeGroup.Key; - IEnumerable tables = typeGroup; + foreach (var typeGroup in AllTableInfos.GroupBy(t => t.DatabaseType)) + { + var dbType = typeGroup.Key; + IEnumerable tables = typeGroup; - var serversByName = tables - .GroupBy(c => c.Server ?? TableInfoServerNode.NullServerNode, StringComparer.CurrentCultureIgnoreCase) - .Select(s => new TableInfoServerNode(s.Key, dbType, s)); + var serversByName = tables + .GroupBy(c => c.Server ?? TableInfoServerNode.NullServerNode, StringComparer.CurrentCultureIgnoreCase) + .Select(s => new TableInfoServerNode(s.Key, dbType, s)); - foreach (var server in serversByName) - { - allServers.Add(server); - AddChildren(server, descendancy.Add(server)); - } - } + foreach (var server in serversByName) + { + allServers.Add(server); + AddChildren(server, descendancy.Add(server)); + } + } - //create the server nodes - AllServers = allServers.ToArray(); + //create the server nodes + AllServers = allServers.ToArray(); - //record the fact that all the servers are children of the all servers node - AddToDictionaries(new HashSet(AllServers), descendancy); - } + //record the fact that all the servers are children of the all servers node + AddToDictionaries(new HashSet(AllServers), descendancy); + } - private void AddChildren(AllDataAccessCredentialsNode allDataAccessCredentialsNode) - { - var children = new HashSet(); + private void AddChildren(AllDataAccessCredentialsNode allDataAccessCredentialsNode) + { + var children = new HashSet(); - var isKeyMissing = false; - if (_catalogueRepository.EncryptionManager is PasswordEncryptionKeyLocation keyLocation) - isKeyMissing = string.IsNullOrWhiteSpace(keyLocation.GetKeyFileLocation()); + var isKeyMissing = false; + if (_catalogueRepository.EncryptionManager is PasswordEncryptionKeyLocation keyLocation) + isKeyMissing = string.IsNullOrWhiteSpace(keyLocation.GetKeyFileLocation()); - children.Add(new DecryptionPrivateKeyNode(isKeyMissing)); + children.Add(new DecryptionPrivateKeyNode(isKeyMissing)); - foreach (var creds in AllDataAccessCredentials) - children.Add(creds); + foreach (var creds in AllDataAccessCredentials) + children.Add(creds); - AddToDictionaries(children, new DescendancyList(allDataAccessCredentialsNode)); - } + AddToDictionaries(children, new DescendancyList(allDataAccessCredentialsNode)); + } - private void AddChildren(AllANOTablesNode anoTablesNode) - { - AddToDictionaries(new HashSet(AllANOTables), new DescendancyList(anoTablesNode)); - } + private void AddChildren(AllANOTablesNode anoTablesNode) + { + AddToDictionaries(new HashSet(AllANOTables), new DescendancyList(anoTablesNode)); + } - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); - //add catalogues in folder - foreach (var c in folder.ChildObjects) AddChildren(c, descendancy.Add(c)); + //add catalogues in folder + foreach (var c in folder.ChildObjects) AddChildren(c, descendancy.Add(c)); - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); - //add loads in folder - foreach (var lmd in folder.ChildObjects.Where(lmd => lmd.RootLoadMetadata_ID == null).ToArray()) AddChildren(lmd, descendancy.Add(lmd)); - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } + //add loads in folder + foreach (var lmd in folder.ChildObjects.Where(lmd => lmd.RootLoadMetadata_ID == null).ToArray()) AddChildren(lmd, descendancy.Add(lmd)); + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); - //add loads in folder - foreach (var ds in folder.ChildObjects) AddChildren(ds, descendancy.Add(ds)); + //add loads in folder + foreach (var ds in folder.ChildObjects) AddChildren(ds, descendancy.Add(ds)); - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); - - - //add cics in folder - foreach (var cic in folder.ChildObjects) AddChildren(cic, descendancy.Add(cic)); - - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } - - private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) - { - var childObjects = new List(); - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - - #region Load Metadata - - private void AddChildren(LoadMetadata lmd, DescendancyList descendancy, bool includeSchedule = true, bool includeCatalogues = true, bool includeVersions = true) - { - var childObjects = new List(); - - if (lmd.OverrideRAWServer_ID.HasValue) - { - var server = AllExternalServers.Single(s => s.ID == lmd.OverrideRAWServer_ID.Value); - var usage = new OverrideRawServerNode(lmd, server); - childObjects.Add(usage); - } - if (includeSchedule) - { - var allSchedulesNode = new LoadMetadataScheduleNode(lmd); - AddChildren(allSchedulesNode, descendancy.Add(allSchedulesNode)); - childObjects.Add(allSchedulesNode); - } - - if (includeCatalogues) - { - var allCataloguesNode = new AllCataloguesUsedByLoadMetadataNode(lmd); - AddChildren(allCataloguesNode, descendancy.Add(allCataloguesNode)); - childObjects.Add(allCataloguesNode); - } - - var processTasksNode = new AllProcessTasksUsedByLoadMetadataNode(lmd); - AddChildren(processTasksNode, descendancy.Add(processTasksNode)); - childObjects.Add(processTasksNode); - - if (includeVersions) - { - var versionsNode = new LoadMetadataVersionNode(lmd); - AddChildren(versionsNode, descendancy.Add(versionsNode)); - childObjects.Add(versionsNode); - } - - childObjects.Add(new LoadDirectoryNode(lmd)); - - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - private void AddChildren(LoadMetadataScheduleNode allSchedulesNode, DescendancyList descendancy) - { - var childObjects = new HashSet(); - - var lmd = allSchedulesNode.LoadMetadata; - - foreach (var lp in AllLoadProgresses.Where(p => p.LoadMetadata_ID == lmd.ID)) - { - AddChildren(lp, descendancy.Add(lp)); - childObjects.Add(lp); - } - - if (childObjects.Any()) - AddToDictionaries(childObjects, descendancy); - } - - private void AddChildren(LoadProgress loadProgress, DescendancyList descendancy) - { - var cacheProgresses = AllCacheProgresses.Where(cp => cp.LoadProgress_ID == loadProgress.ID).ToArray(); - - foreach (var cacheProgress in cacheProgresses) - AddChildren(cacheProgress, descendancy.Add(cacheProgress)); - - if (cacheProgresses.Any()) - AddToDictionaries(new HashSet(cacheProgresses), descendancy); - } - - private void AddChildren(CacheProgress cacheProgress, DescendancyList descendancy) - { - var children = new HashSet(); - - if (cacheProgress.PermissionWindow_ID != null) - { - var window = AllPermissionWindows.Single(w => w.ID == cacheProgress.PermissionWindow_ID); - var windowNode = new PermissionWindowUsedByCacheProgressNode(cacheProgress, window, true); - - children.Add(windowNode); - } - - if (children.Any()) - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllProcessTasksUsedByLoadMetadataNode allProcessTasksUsedByLoadMetadataNode, - DescendancyList descendancy) - { - var childObjects = new HashSet(); - - var lmd = allProcessTasksUsedByLoadMetadataNode.LoadMetadata; - childObjects.Add(new LoadStageNode(lmd, LoadStage.GetFiles)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.Mounting)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustRaw)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustStaging)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.PostLoad)); - - foreach (LoadStageNode node in childObjects) - AddChildren(node, descendancy.Add(node)); - - AddToDictionaries(childObjects, descendancy); - } - - private void AddChildren(LoadStageNode loadStageNode, DescendancyList descendancy) - { - var tasks = AllProcessTasks.Where( - p => p.LoadMetadata_ID == loadStageNode.LoadMetadata.ID && p.LoadStage == loadStageNode.LoadStage) - .OrderBy(o => o.Order).ToArray(); - - foreach (var processTask in tasks) - AddChildren(processTask, descendancy.Add(processTask)); - - if (tasks.Any()) - AddToDictionaries(new HashSet(tasks), descendancy); - } - - private void AddChildren(ProcessTask procesTask, DescendancyList descendancy) - { - var args = AllProcessTasksArguments.Where( - a => a.ProcessTask_ID == procesTask.ID).ToArray(); - - if (args.Any()) - AddToDictionaries(new HashSet(args), descendancy); - } - - private void AddChildren(LoadMetadataVersionNode LoadMetadataVersionNode, DescendancyList descendancy) - { - LoadMetadataVersionNode.LoadMetadataVersions = AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID == LoadMetadataVersionNode.LoadMetadata.ID).ToList(); - var childObjects = new List(); - - foreach (var lmd in LoadMetadataVersionNode.LoadMetadataVersions) - { - AddChildren(lmd, descendancy.Add(lmd), false, false, false); - childObjects.Add(lmd); - } - AddToDictionaries(new HashSet(childObjects), descendancy); - - } - - private void AddChildren(AllCataloguesUsedByLoadMetadataNode allCataloguesUsedByLoadMetadataNode, - DescendancyList descendancy) - { - var loadMetadataId = allCataloguesUsedByLoadMetadataNode.LoadMetadata.ID; - var linkedCatalogueIDs = AllLoadMetadataLinkage.Where(link => link.LoadMetadataID == loadMetadataId).Select(static link => link.CatalogueID); - var usedCatalogues = linkedCatalogueIDs.Select(catalogueId => AllCatalogues.FirstOrDefault(c => c.ID == catalogueId)).Where(static foundCatalogue => foundCatalogue is not null).ToList(); - allCataloguesUsedByLoadMetadataNode.UsedCatalogues = usedCatalogues; - var childObjects = usedCatalogues.Select(foundCatalogue => new CatalogueUsedByLoadMetadataNode(allCataloguesUsedByLoadMetadataNode.LoadMetadata, foundCatalogue)).Cast().ToHashSet(); - - AddToDictionaries(childObjects, descendancy); - } - - #endregion - - protected void AddChildren(Catalogue c, DescendancyList descendancy) - { - var childObjects = new List(); - - var catalogueAggregates = AllAggregateConfigurations.Where(a => a.Catalogue_ID == c.ID).ToArray(); - var cohortAggregates = catalogueAggregates.Where(a => a.IsCohortIdentificationAggregate).ToArray(); - var regularAggregates = catalogueAggregates.Except(cohortAggregates).ToArray(); - - //get all the CatalogueItems for this Catalogue (TryGet because Catalogue may not have any items - var cis = _catalogueToCatalogueItems.TryGetValue(c.ID, out var result) - ? result.ToArray() - : Array.Empty(); - - //tell the CatalogueItems that we are are their parent - foreach (var ci in cis) - ci.InjectKnown(c); - - // core includes project specific which basically means the same thing - var core = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Core || - ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.ProjectSpecific) - , ExtractionCategory.Core); - - c.InjectKnown(cis); - - var deprecated = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Deprecated), - ExtractionCategory.Deprecated); - var special = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.SpecialApprovalRequired), - ExtractionCategory.SpecialApprovalRequired); - var intern = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Internal), - ExtractionCategory.Internal); - var supplemental = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Supplemental), - ExtractionCategory.Supplemental); - var notExtractable = new CatalogueItemsNode(c, cis.Where(ci => ci.ExtractionInformation == null), null); - - AddChildren(core, descendancy.Add(core)); - childObjects.Add(core); - - foreach (var optional in new[] { deprecated, special, intern, supplemental, notExtractable }) - if (optional.CatalogueItems.Any()) - { - AddChildren(optional, descendancy.Add(optional)); - childObjects.Add(optional); - } - - //do we have any foreign key fields into this lookup table - var lookups = AllLookups.Where(l => c.CatalogueItems.Any(ci => ci.ColumnInfo_ID == l.ForeignKey_ID)).ToArray(); - - var docs = AllSupportingDocuments.Where(d => d.Catalogue_ID == c.ID).ToArray(); - var sql = AllSupportingSQL.Where(d => d.Catalogue_ID == c.ID).ToArray(); - - //if there are supporting documents or supporting sql files then add documentation node - if (docs.Any() || sql.Any()) - { - var documentationNode = new DocumentationNode(c, docs, sql); - - //add the documentations node - childObjects.Add(documentationNode); - - //record the children - AddToDictionaries(new HashSet(docs.Cast().Union(sql)), descendancy.Add(documentationNode)); - } - - if (lookups.Any()) - { - var lookupsNode = new CatalogueLookupsNode(c, lookups); - //add the documentations node - childObjects.Add(lookupsNode); - - - //record the children - AddToDictionaries(new HashSet(lookups.Select(l => new CatalogueLookupUsageNode(c, l))), - descendancy.Add(lookupsNode)); - } - - if (regularAggregates.Any()) - { - var aggregatesNode = new AggregatesNode(c, regularAggregates); - childObjects.Add(aggregatesNode); - - var nodeDescendancy = descendancy.Add(aggregatesNode); - AddToDictionaries(new HashSet(regularAggregates), nodeDescendancy); - - foreach (var regularAggregate in regularAggregates) - AddChildren(regularAggregate, nodeDescendancy.Add(regularAggregate)); - } - - //finalise - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - private void InjectCatalogueItems() - { - foreach (var ci in AllCatalogueItems) - if (_extractionInformationsByCatalogueItem.TryGetValue(ci.ID, out var ei)) - ci.InjectKnown(ei); - else - ci.InjectKnown((ExtractionInformation)null); - } - - private void AddChildren(CatalogueItemsNode node, DescendancyList descendancyList) - { - AddToDictionaries(new HashSet(node.CatalogueItems), descendancyList); - - foreach (var ci in node.CatalogueItems) - AddChildren(ci, descendancyList.Add(ci)); - } - - private void AddChildren(AggregateConfiguration aggregateConfiguration, DescendancyList descendancy) - { - var childrenObjects = new HashSet(); - - var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(aggregateConfiguration)).Cast() - .ToArray(); - - foreach (var p in parameters) - childrenObjects.Add(p); - - // show the dimensions in the tree - foreach (var dim in aggregateConfiguration.AggregateDimensions) childrenObjects.Add(dim); - - // show the axis (if any) in the tree. If there are multiple axis in this tree then that is bad but maybe the user can delete one of them to fix the situation - foreach (var axis in AllAggregateContinuousDateAxis.Where(a => - aggregateConfiguration.AggregateDimensions.Any(d => d.ID == a.AggregateDimension_ID))) - childrenObjects.Add(axis); - - //we can step into this twice, once via Catalogue children and once via CohortIdentificationConfiguration children - //if we get in via Catalogue children then descendancy will be Ignore=true we don't end up emphasising into CatalogueCollectionUI when - //really user wants to see it in CohortIdentificationCollectionUI - if (aggregateConfiguration.RootFilterContainer_ID != null) - { - var container = AllAggregateContainersDictionary[(int)aggregateConfiguration.RootFilterContainer_ID]; - - AddChildren(container, descendancy.Add(container)); - childrenObjects.Add(container); - } - - AddToDictionaries(childrenObjects, descendancy); - } - - private void AddChildren(AggregateFilterContainer container, DescendancyList descendancy) - { - var childrenObjects = new List(); - - var subcontainers = _aggregateFilterManager.GetSubContainers(container); - var filters = _aggregateFilterManager.GetFilters(container); - - foreach (AggregateFilterContainer subcontainer in subcontainers) - { - //one of our children is this subcontainer - childrenObjects.Add(subcontainer); - - //but also document its children - AddChildren(subcontainer, descendancy.Add(subcontainer)); - } - - //also add the filters for the container - foreach (var f in filters) - { - // for filters add the parameters under them - AddChildren((AggregateFilter)f, descendancy.Add(f)); - childrenObjects.Add(f); - } - - //add our children to the dictionary - AddToDictionaries(new HashSet(childrenObjects), descendancy); - } - - private void AddChildren(AggregateFilter f, DescendancyList descendancy) - { - AddToDictionaries(new HashSet(AllAggregateFilterParameters.Where(p => p.AggregateFilter_ID == f.ID)), - descendancy); - } - - private void AddChildren(CatalogueItem ci, DescendancyList descendancy) - { - var childObjects = new List(); - - var ei = ci.ExtractionInformation; - if (ei != null) - { - childObjects.Add(ei); - AddChildren(ei, descendancy.Add(ei)); - } - else - { - ci.InjectKnown( - (ExtractionInformation)null); // we know the CatalogueItem has no ExtractionInformation child because it's not in the dictionary - } - - if (ci.ColumnInfo_ID.HasValue && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) - childObjects.Add(new LinkedColumnInfoNode(ci, col)); - - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - private void AddChildren(ExtractionInformation extractionInformation, DescendancyList descendancy) - { - var children = new HashSet(); - - foreach (var filter in AllCatalogueFilters.Where(f => f.ExtractionInformation_ID == extractionInformation.ID)) - { - //add the filter as a child of the - children.Add(filter); - AddChildren(filter, descendancy.Add(filter)); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(ExtractionFilter filter, DescendancyList descendancy) - { - var children = new HashSet(); - var parameters = AllCatalogueParameters.Where(p => p.ExtractionFilter_ID == filter.ID).ToArray(); - var parameterSets = AllCatalogueValueSets.Where(vs => vs.ExtractionFilter_ID == filter.ID).ToArray(); - - filter.InjectKnown(parameterSets); - - foreach (var p in parameters) - children.Add(p); - - foreach (var set in parameterSets) - { - children.Add(set); - AddChildren(set, descendancy.Add(set), parameters); - } - - if (children.Any()) - AddToDictionaries(children, descendancy); - } - - private void AddChildren(ExtractionFilterParameterSet set, DescendancyList descendancy, - ExtractionFilterParameter[] filterParameters) - { - var children = new HashSet(); - - foreach (var setValue in AllCatalogueValueSetValues.Where(v => v.ExtractionFilterParameterSet_ID == set.ID)) - { - setValue.InjectKnown(filterParameters.SingleOrDefault(p => p.ID == setValue.ExtractionFilterParameter_ID)); - children.Add(setValue); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(CohortIdentificationConfiguration cic, DescendancyList descendancy) - { - var children = new HashSet(); - - //it has an associated query cache - if (cic.QueryCachingServer_ID != null) - children.Add(new QueryCacheUsedByCohortIdentificationNode(cic, - AllExternalServers.Single(s => s.ID == cic.QueryCachingServer_ID))); - - var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(cic)).Cast().ToArray(); - foreach (var p in parameters) children.Add(p); - - //if it has a root container - if (cic.RootCohortAggregateContainer_ID != null) - { - var container = AllCohortAggregateContainers.Single(c => c.ID == cic.RootCohortAggregateContainer_ID); - AddChildren(container, descendancy.Add(container).SetBetterRouteExists()); - children.Add(container); - } - - //get the patient index tables - var joinableNode = new JoinableCollectionNode(cic, - AllJoinables.Where(j => j.CohortIdentificationConfiguration_ID == cic.ID).ToArray()); - AddChildren(joinableNode, descendancy.Add(joinableNode).SetBetterRouteExists()); - children.Add(joinableNode); - - AddToDictionaries(children, descendancy.SetBetterRouteExists()); - } - - private void AddChildren(JoinableCollectionNode joinablesNode, DescendancyList descendancy) - { - var children = new HashSet(); - - foreach (var joinable in joinablesNode.Joinables) - try - { - var agg = AllAggregateConfigurations.Single(ac => ac.ID == joinable.AggregateConfiguration_ID); - ForceAggregateNaming(agg, descendancy); - children.Add(agg); - - //it's no longer an orphan because it's in a known cic (as a patient index table) - OrphanAggregateConfigurations.Remove(agg); - - AddChildren(agg, descendancy.Add(agg)); - } - catch (Exception e) - { - throw new Exception( - $"JoinableCohortAggregateConfiguration (patient index table) object (ID={joinable.ID}) references AggregateConfiguration_ID {joinable.AggregateConfiguration_ID} but that AggregateConfiguration was not found", - e); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(CohortAggregateContainer container, DescendancyList descendancy) - { - //get subcontainers - var subcontainers = _cohortContainerManager.GetChildren(container).OfType().ToList(); - - //if there are subcontainers - foreach (var subcontainer in subcontainers) - AddChildren(subcontainer, descendancy.Add(subcontainer)); - - //get our configurations - var configurations = _cohortContainerManager.GetChildren(container).OfType().ToList(); - - //record the configurations children including full descendancy - foreach (var configuration in configurations) - { - ForceAggregateNaming(configuration, descendancy); - AddChildren(configuration, descendancy.Add(configuration)); - - //it's no longer an orphan because it's in a known cic - OrphanAggregateConfigurations.Remove(configuration); - } - - //all our children (containers and aggregates) - //children are all aggregates and containers at the current hierarchy level in order - var children = subcontainers.Union(configurations.Cast()).OrderBy(o => o.Order).ToList(); - - AddToDictionaries(new HashSet(children), descendancy); - } - - private void ForceAggregateNaming(AggregateConfiguration configuration, DescendancyList descendancy) - { - //configuration has the wrong name - if (!configuration.IsCohortIdentificationAggregate) - { - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( - $"Had to fix naming of configuration '{configuration}' because it didn't start with correct cic prefix", - CheckResult.Warning)); - descendancy.Parents.OfType().Single() - .EnsureNamingConvention(configuration); - configuration.SaveToDatabase(); - } - } - - private void AddChildren(TableInfoServerNode serverNode, DescendancyList descendancy) - { - //add empty hashset - var children = new HashSet(); - - var databases = - serverNode.Tables.GroupBy( - k => k.Database ?? TableInfoDatabaseNode.NullDatabaseNode, StringComparer.CurrentCultureIgnoreCase) - .Select(g => new TableInfoDatabaseNode(g.Key, serverNode, g)); - - foreach (var db in databases) - { - children.Add(db); - AddChildren(db, descendancy.Add(db)); - } - - //now we have recorded all the children add them with descendancy - AddToDictionaries(children, descendancy); - } - - private void AddChildren(TableInfoDatabaseNode dbNode, DescendancyList descendancy) - { - //add empty hashset - var children = new HashSet(); - - foreach (var t in dbNode.Tables) - { - //record the children of the table infos (mostly column infos) - children.Add(t); - - //the all servers node=>the TableInfoServerNode => the t - AddChildren(t, descendancy.Add(t)); - } - - //now we have recorded all the children add them with descendancy - AddToDictionaries(children, descendancy); - } - - private void AddChildren(TableInfo tableInfo, DescendancyList descendancy) - { - //add empty hashset - var children = new HashSet(); - - //if the table has an identifier dump listed - if (tableInfo.IdentifierDumpServer_ID != null) - { - //if there is a dump (e.g. for dilution and dumping - not appearing in the live table) - var server = AllExternalServers.Single(s => s.ID == tableInfo.IdentifierDumpServer_ID.Value); - - children.Add(new IdentifierDumpServerUsageNode(tableInfo, server)); - } - - //get the discarded columns in this table - var discardedCols = new HashSet(AllPreLoadDiscardedColumns.Where(c => c.TableInfo_ID == tableInfo.ID)); - - //tell the column who their parent is so they don't need to look up the database - foreach (PreLoadDiscardedColumn discardedCol in discardedCols) - discardedCol.InjectKnown(tableInfo); - - //if there are discarded columns - if (discardedCols.Any()) - { - var identifierDumpNode = new PreLoadDiscardedColumnsNode(tableInfo); - - //record that the usage is a child of TableInfo - children.Add(identifierDumpNode); - - //record that the discarded columns are children of identifier dump usage node - AddToDictionaries(discardedCols, descendancy.Add(identifierDumpNode)); - } - - //if it is a table valued function - if (tableInfo.IsTableValuedFunction) - { - //that has parameters - var parameters = tableInfo.GetAllParameters(); - - foreach (var p in parameters) children.Add(p); - } - - //next add the column infos - if (TableInfosToColumnInfos.TryGetValue(tableInfo.ID, out var result)) - foreach (var c in result) - { - children.Add(c); - c.InjectKnown(tableInfo); - AddChildren(c, descendancy.Add(c).SetBetterRouteExists()); - } - - //finally add any credentials objects - if (AllDataAccessCredentialUsages.TryGetValue(tableInfo, out var nodes)) - foreach (var node in nodes) - children.Add(node); - - //now we have recorded all the children add them with descendancy via the TableInfo descendancy - AddToDictionaries(children, descendancy); - } - - private void AddChildren(ColumnInfo columnInfo, DescendancyList descendancy) - { - var lookups = AllLookups.Where(l => l.Description_ID == columnInfo.ID).ToArray(); - var joinInfos = AllJoinInfos.Where(j => j.PrimaryKey_ID == columnInfo.ID); - - var children = new HashSet(); - - foreach (var l in lookups) - children.Add(l); - - foreach (var j in joinInfos) - children.Add(j); - - if (children.Any()) - AddToDictionaries(children, descendancy); - } - - - protected void AddToDictionaries(HashSet children, DescendancyList list) - { - if (list.IsEmpty) - throw new ArgumentException("DescendancyList cannot be empty", nameof(list)); - - //document that the last parent has these as children - var parent = list.Last(); - - _childDictionary.AddOrUpdate(parent, - children, (p, s) => children); - - //now document the entire parent order to reach each child object i.e. 'Root=>Grandparent=>Parent' is how you get to 'Child' - foreach (var o in children) - _descendancyDictionary.AddOrUpdate(o, list, (k, v) => HandleDescendancyCollision(k, v, list)); - - - foreach (var masquerader in children.OfType()) - { - var key = masquerader.MasqueradingAs(); - - if (!AllMasqueraders.ContainsKey(key)) - AllMasqueraders.AddOrUpdate(key, new HashSet(), (o, set) => set); - - lock (AllMasqueraders) - { - AllMasqueraders[key].Add(masquerader); - } - } - } - - private static DescendancyList HandleDescendancyCollision(object key, DescendancyList oldRoute, - DescendancyList newRoute) - { - //if the new route is the best best - if (newRoute.NewBestRoute && !oldRoute.NewBestRoute) - return newRoute; - - // If the new one is marked BetterRouteExists just throw away the new one - return newRoute.BetterRouteExists ? oldRoute : newRoute; - // If in doubt use the newest one - } - - private HashSet GetAllObjects() - { - //anything which has children or is a child of someone else (distinct because HashSet) - return new HashSet(_childDictionary.SelectMany(kvp => kvp.Value).Union(_childDictionary.Keys)); - } - - public virtual object[] GetChildren(object model) - { - lock (WriteLock) - { -; //if we have a record of any children in the child dictionary for the parent model object - if (_childDictionary.TryGetValue(model, out var cached)) - return cached.OrderBy(static o => o.ToString()).ToArray(); - - return model switch - { - //if they want the children of a Pipeline (which we don't track) just serve the components - Pipeline p => p.PipelineComponents.ToArray(), - //if they want the children of a PipelineComponent (which we don't track) just serve the arguments - PipelineComponent pc => pc.PipelineComponentArguments.ToArray(), - _ => Array.Empty() - }; - } - } - - public IEnumerable GetAllObjects(Type type, bool unwrapMasqueraders) - { - lock (WriteLock) - { - //things that are a match on Type but not IMasqueradeAs - var exactMatches = GetAllSearchables().Keys.Where(t => t is not IMasqueradeAs).Where(type.IsInstanceOfType); - - //Union the unwrapped masqueraders - return unwrapMasqueraders - ? exactMatches.Union( - AllMasqueraders - .Select(kvp => kvp.Key) - .OfType() - .Where(type.IsInstanceOfType)) - .Distinct() - : exactMatches; - } - } - - public DescendancyList GetDescendancyListIfAnyFor(object model) - { - lock (WriteLock) - { - return _descendancyDictionary.GetValueOrDefault(model); - } - } - - - public object GetRootObjectOrSelf(object objectToEmphasise) - { - lock (WriteLock) - { - var descendancy = GetDescendancyListIfAnyFor(objectToEmphasise); - - return descendancy != null && descendancy.Parents.Any() ? descendancy.Parents[0] : objectToEmphasise; - } - } - - - public virtual Dictionary GetAllSearchables() - { - lock (WriteLock) - { - var toReturn = new Dictionary(); - - foreach (var kvp in _descendancyDictionary.Where(kvp => kvp.Key is IMapsDirectlyToDatabaseTable)) - toReturn.Add((IMapsDirectlyToDatabaseTable)kvp.Key, kvp.Value); - - return toReturn; - } - } - - public IEnumerable GetAllChildrenRecursively(object o) - { - lock (WriteLock) - { - var toReturn = new List(); - - foreach (var child in GetChildren(o)) - { - toReturn.Add(child); - toReturn.AddRange(GetAllChildrenRecursively(child)); - } - - return toReturn; - } - } - - /// - /// Asks all plugins to provide the child objects for every object we have found so far. This method is recursive, call it with null the first time to use all objects. It will then - /// call itself with all the new objects that were sent back by the plugin (so that new objects found can still have children). - /// - /// - protected void GetPluginChildren(HashSet objectsToAskAbout = null) - { - lock (WriteLock) - { - var newObjectsFound = new HashSet(); - - var sw = new Stopwatch(); - - var providers = _pluginChildProviders.Except(_blockedPlugins).ToArray(); - - //for every object found so far - if (providers.Any()) - foreach (var o in objectsToAskAbout ?? GetAllObjects()) - //for every plugin loaded (that is not forbidlisted) - foreach (var plugin in providers) - //ask about the children - try - { - sw.Restart(); - //otherwise ask plugin what its children are - var pluginChildren = plugin.GetChildren(o); - - //if the plugin takes too long to respond we need to stop - if (sw.ElapsedMilliseconds > 1000) - { - _blockedPlugins.Add(plugin); - throw new Exception( - $"Plugin '{plugin}' was forbidlisted for taking too long to respond to GetChildren(o) where o was a '{o.GetType().Name}' ('{o}')"); - } - - //it has children - if (pluginChildren != null && pluginChildren.Any()) - { - //get the descendancy of the parent - var parentDescendancy = GetDescendancyListIfAnyFor(o); - var newDescendancy = parentDescendancy == null - ? new DescendancyList(new[] { o }) - : //if the parent is a root level object start a new descendancy list from it - parentDescendancy - .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one - newDescendancy = - parentDescendancy - .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one - - //record that - foreach (var pluginChild in pluginChildren) - { - //if the parent didn't have any children before - if (!_childDictionary.ContainsKey(o)) - _childDictionary.AddOrUpdate(o, new HashSet(), - (o1, set) => set); //it does now - - - //add us to the parent objects child collection - _childDictionary[o].Add(pluginChild); - - //add to the child collection of the parent object kvp.Key - _descendancyDictionary.AddOrUpdate(pluginChild, newDescendancy, - (s, e) => newDescendancy); - - //we have found a new object so we must ask other plugins about it (chances are a plugin will have a whole tree of sub objects) - newObjectsFound.Add(pluginChild); - } - } - } - catch (Exception e) - { - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail, e)); - } - - if (newObjectsFound.Any()) - GetPluginChildren(newObjectsFound); - } - } - - public IEnumerable GetMasqueradersOf(object o) - { - lock (WriteLock) - { - return AllMasqueraders.TryGetValue(o, out var result) ? result : Array.Empty(); - } - } - - protected T[] GetAllObjects(IRepository repository) where T : IMapsDirectlyToDatabaseTable - { - lock (WriteLock) - { - return repository.GetAllObjects(); - } - } - - - protected void AddToReturnSearchablesWithNoDecendancy( - Dictionary toReturn, - IEnumerable toAdd) - { - lock (WriteLock) - { - foreach (var m in toAdd) - toReturn.Add(m, null); - } - } - - public virtual void UpdateTo(ICoreChildProvider other) - { - ArgumentNullException.ThrowIfNull(other); - - if (other is not CatalogueChildProvider otherCat) - throw new NotSupportedException( - $"Did not know how to UpdateTo ICoreChildProvider of type {other.GetType().Name}"); - - AllLoadMetadatas = otherCat.AllLoadMetadatas; - AllProcessTasks = otherCat.AllProcessTasks; - AllProcessTasksArguments = otherCat.AllProcessTasksArguments; - AllLoadProgresses = otherCat.AllLoadProgresses; - AllCacheProgresses = otherCat.AllCacheProgresses; - AllPermissionWindows = otherCat.AllPermissionWindows; - AllCatalogues = otherCat.AllCatalogues; - AllCataloguesDictionary = otherCat.AllCataloguesDictionary; - AllSupportingDocuments = otherCat.AllSupportingDocuments; - AllSupportingSQL = otherCat.AllSupportingSQL; - _childDictionary = otherCat._childDictionary; - _descendancyDictionary = otherCat._descendancyDictionary; - _catalogueToCatalogueItems = otherCat._catalogueToCatalogueItems; - AllCatalogueItemsDictionary = otherCat.AllCatalogueItemsDictionary; - _allColumnInfos = otherCat._allColumnInfos; - AllAggregateConfigurations = otherCat.AllAggregateConfigurations; - AllAggregateDimensions = otherCat.AllAggregateDimensions; - AllAggregateContinuousDateAxis = otherCat.AllAggregateContinuousDateAxis; - AllRDMPRemotesNode = otherCat.AllRDMPRemotesNode; - AllRemoteRDMPs = otherCat.AllRemoteRDMPs; - AllDashboardsNode = otherCat.AllDashboardsNode; - AllDashboards = otherCat.AllDashboards; - AllObjectSharingNode = otherCat.AllObjectSharingNode; - AllImports = otherCat.AllImports; - AllExports = otherCat.AllExports; - AllStandardRegexesNode = otherCat.AllStandardRegexesNode; - AllPipelinesNode = otherCat.AllPipelinesNode; - OtherPipelinesNode = otherCat.OtherPipelinesNode; - AllPipelines = otherCat.AllPipelines; - AllPipelineComponents = otherCat.AllPipelineComponents; - AllPipelineComponentsArguments = otherCat.AllPipelineComponentsArguments; - AllStandardRegexes = otherCat.AllStandardRegexes; - AllANOTablesNode = otherCat.AllANOTablesNode; - AllANOTables = otherCat.AllANOTables; - AllExternalServers = otherCat.AllExternalServers; - AllServers = otherCat.AllServers; - AllTableInfos = otherCat.AllTableInfos; - AllDataAccessCredentialsNode = otherCat.AllDataAccessCredentialsNode; - AllExternalServersNode = otherCat.AllExternalServersNode; - AllServersNode = otherCat.AllServersNode; - AllDataAccessCredentials = otherCat.AllDataAccessCredentials; - AllDataAccessCredentialUsages = otherCat.AllDataAccessCredentialUsages; - TableInfosToColumnInfos = otherCat.TableInfosToColumnInfos; - AllColumnInfos = otherCat.AllColumnInfos; - AllPreLoadDiscardedColumns = otherCat.AllPreLoadDiscardedColumns; - AllLookups = otherCat.AllLookups; - AllJoinInfos = otherCat.AllJoinInfos; - AllAnyTableParameters = otherCat.AllAnyTableParameters; - AllMasqueraders = otherCat.AllMasqueraders; - AllExtractionInformationsDictionary = otherCat.AllExtractionInformationsDictionary; - _pluginChildProviders = otherCat._pluginChildProviders; - AllPermissionWindowsNode = otherCat.AllPermissionWindowsNode; - LoadMetadataRootFolder = otherCat.LoadMetadataRootFolder; - CatalogueRootFolder = otherCat.CatalogueRootFolder; - CohortIdentificationConfigurationRootFolder = otherCat.CohortIdentificationConfigurationRootFolder; - AllConnectionStringKeywordsNode = otherCat.AllConnectionStringKeywordsNode; - AllConnectionStringKeywords = otherCat.AllConnectionStringKeywords; - AllAggregateContainersDictionary = otherCat.AllAggregateContainersDictionary; - AllAggregateFilters = otherCat.AllAggregateFilters; - AllAggregateFilterParameters = otherCat.AllAggregateFilterParameters; - AllCohortIdentificationConfigurations = otherCat.AllCohortIdentificationConfigurations; - AllCohortAggregateContainers = otherCat.AllCohortAggregateContainers; - AllJoinables = otherCat.AllJoinables; - AllJoinUses = otherCat.AllJoinUses; - AllGovernanceNode = otherCat.AllGovernanceNode; - AllGovernancePeriods = otherCat.AllGovernancePeriods; - AllGovernanceDocuments = otherCat.AllGovernanceDocuments; - GovernanceCoverage = otherCat.GovernanceCoverage; - AllJoinableCohortAggregateConfigurationUse = otherCat.AllJoinableCohortAggregateConfigurationUse; - AllPluginsNode = otherCat.AllPluginsNode; - PipelineUseCases = otherCat.PipelineUseCases; - OrphanAggregateConfigurationsNode = otherCat.OrphanAggregateConfigurationsNode; - TemplateAggregateConfigurationsNode = otherCat.TemplateAggregateConfigurationsNode; - AllCatalogueParameters = otherCat.AllCatalogueParameters; - AllCatalogueValueSets = otherCat.AllCatalogueValueSets; - AllCatalogueValueSetValues = otherCat.AllCatalogueValueSetValues; - OrphanAggregateConfigurations = otherCat.OrphanAggregateConfigurations; - } - - public virtual bool SelectiveRefresh(IMapsDirectlyToDatabaseTable databaseEntity) - { - ProgressStopwatch.Restart(); - - return databaseEntity switch - { - AggregateFilterParameter afp => SelectiveRefresh(afp.AggregateFilter), - AggregateFilter af => SelectiveRefresh(af), - AggregateFilterContainer afc => SelectiveRefresh(afc), - CohortAggregateContainer cac => SelectiveRefresh(cac), - ExtractionInformation ei => SelectiveRefresh(ei), - CatalogueItem ci => SelectiveRefresh(ci), - _ => false - }; - } - - - public bool SelectiveRefresh(CatalogueItem ci) - { - var descendancy = GetDescendancyListIfAnyFor(ci.Catalogue); - if (descendancy == null) return false; - - FetchCatalogueItems(); - FetchExtractionInformations(); - AddChildren(ci.Catalogue, descendancy.Add(ci.Catalogue)); - return true; - } - - public bool SelectiveRefresh(ExtractionInformation ei) - { - var descendancy = GetDescendancyListIfAnyFor(ei); - var cata = descendancy?.Parents.OfType().LastOrDefault() ?? ei.CatalogueItem.Catalogue; - - if (cata == null) - return false; - - var cataDescendancy = GetDescendancyListIfAnyFor(cata); - - if (cataDescendancy == null) - return false; - - FetchCatalogueItems(); - - foreach (var ci in AllCatalogueItems.Where(ci => ci.ID == ei.CatalogueItem_ID)) ci.ClearAllInjections(); - - // property changes or deleting the ExtractionInformation - FetchExtractionInformations(); - - // refresh the Catalogue - AddChildren(cata, cataDescendancy.Add(cata)); - return true; - } - - public bool SelectiveRefresh(CohortAggregateContainer container) - { - var parentContainer = container.GetParentContainerIfAny(); - if (parentContainer != null) - { - var descendancy = GetDescendancyListIfAnyFor(parentContainer); - - if (descendancy != null) - { - BuildAggregateConfigurations(); - - BuildCohortCohortAggregateContainers(); - AddChildren(parentContainer, descendancy.Add(parentContainer)); - return true; - } - } - - var cic = container.GetCohortIdentificationConfiguration(); - - if (cic != null) - { - var descendancy = GetDescendancyListIfAnyFor(cic); - - if (descendancy != null) - { - BuildAggregateConfigurations(); - BuildCohortCohortAggregateContainers(); - AddChildren(cic, descendancy.Add(cic)); - return true; - } - } - - return false; - } - - public bool SelectiveRefresh(AggregateFilter f) - { - var knownContainer = GetDescendancyListIfAnyFor(f.FilterContainer); - if (knownContainer == null) return false; - - BuildAggregateFilterContainers(); - AddChildren((AggregateFilterContainer)f.FilterContainer, knownContainer.Add(f.FilterContainer)); - return true; - } + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); + + + //add cics in folder + foreach (var cic in folder.ChildObjects) AddChildren(cic, descendancy.Add(cic)); + + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } + + private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) + { + var childObjects = new List(); + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + + #region Load Metadata + + private void AddChildren(LoadMetadata lmd, DescendancyList descendancy, bool includeSchedule = true, bool includeCatalogues = true, bool includeVersions = true) + { + var childObjects = new List(); + + if (lmd.OverrideRAWServer_ID.HasValue) + { + var server = AllExternalServers.Single(s => s.ID == lmd.OverrideRAWServer_ID.Value); + var usage = new OverrideRawServerNode(lmd, server); + childObjects.Add(usage); + } + if (includeSchedule) + { + var allSchedulesNode = new LoadMetadataScheduleNode(lmd); + AddChildren(allSchedulesNode, descendancy.Add(allSchedulesNode)); + childObjects.Add(allSchedulesNode); + } + + if (includeCatalogues) + { + var allCataloguesNode = new AllCataloguesUsedByLoadMetadataNode(lmd); + AddChildren(allCataloguesNode, descendancy.Add(allCataloguesNode)); + childObjects.Add(allCataloguesNode); + } + + var processTasksNode = new AllProcessTasksUsedByLoadMetadataNode(lmd); + AddChildren(processTasksNode, descendancy.Add(processTasksNode)); + childObjects.Add(processTasksNode); + + if (includeVersions) + { + var versionsNode = new LoadMetadataVersionNode(lmd); + AddChildren(versionsNode, descendancy.Add(versionsNode)); + childObjects.Add(versionsNode); + } + + childObjects.Add(new LoadDirectoryNode(lmd)); + + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + private void AddChildren(LoadMetadataScheduleNode allSchedulesNode, DescendancyList descendancy) + { + var childObjects = new HashSet(); + + var lmd = allSchedulesNode.LoadMetadata; + + foreach (var lp in AllLoadProgresses.Where(p => p.LoadMetadata_ID == lmd.ID)) + { + AddChildren(lp, descendancy.Add(lp)); + childObjects.Add(lp); + } + + if (childObjects.Any()) + AddToDictionaries(childObjects, descendancy); + } + + private void AddChildren(LoadProgress loadProgress, DescendancyList descendancy) + { + var cacheProgresses = AllCacheProgresses.Where(cp => cp.LoadProgress_ID == loadProgress.ID).ToArray(); + + foreach (var cacheProgress in cacheProgresses) + AddChildren(cacheProgress, descendancy.Add(cacheProgress)); + + if (cacheProgresses.Any()) + AddToDictionaries(new HashSet(cacheProgresses), descendancy); + } + + private void AddChildren(CacheProgress cacheProgress, DescendancyList descendancy) + { + var children = new HashSet(); + + if (cacheProgress.PermissionWindow_ID != null) + { + var window = AllPermissionWindows.Single(w => w.ID == cacheProgress.PermissionWindow_ID); + var windowNode = new PermissionWindowUsedByCacheProgressNode(cacheProgress, window, true); + + children.Add(windowNode); + } + + if (children.Any()) + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllProcessTasksUsedByLoadMetadataNode allProcessTasksUsedByLoadMetadataNode, + DescendancyList descendancy) + { + var childObjects = new HashSet(); + + var lmd = allProcessTasksUsedByLoadMetadataNode.LoadMetadata; + childObjects.Add(new LoadStageNode(lmd, LoadStage.GetFiles)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.Mounting)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustRaw)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustStaging)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.PostLoad)); + + foreach (LoadStageNode node in childObjects) + AddChildren(node, descendancy.Add(node)); + + AddToDictionaries(childObjects, descendancy); + } + + private void AddChildren(LoadStageNode loadStageNode, DescendancyList descendancy) + { + var tasks = AllProcessTasks.Where( + p => p.LoadMetadata_ID == loadStageNode.LoadMetadata.ID && p.LoadStage == loadStageNode.LoadStage) + .OrderBy(o => o.Order).ToArray(); + + foreach (var processTask in tasks) + AddChildren(processTask, descendancy.Add(processTask)); + + if (tasks.Any()) + AddToDictionaries(new HashSet(tasks), descendancy); + } + + private void AddChildren(ProcessTask procesTask, DescendancyList descendancy) + { + var args = AllProcessTasksArguments.Where( + a => a.ProcessTask_ID == procesTask.ID).ToArray(); + + if (args.Any()) + AddToDictionaries(new HashSet(args), descendancy); + } + + private void AddChildren(LoadMetadataVersionNode LoadMetadataVersionNode, DescendancyList descendancy) + { + LoadMetadataVersionNode.LoadMetadataVersions = AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID == LoadMetadataVersionNode.LoadMetadata.ID).ToList(); + var childObjects = new List(); + + foreach (var lmd in LoadMetadataVersionNode.LoadMetadataVersions) + { + AddChildren(lmd, descendancy.Add(lmd), false, false, false); + childObjects.Add(lmd); + } + AddToDictionaries(new HashSet(childObjects), descendancy); + + } + + private void AddChildren(AllCataloguesUsedByLoadMetadataNode allCataloguesUsedByLoadMetadataNode, + DescendancyList descendancy) + { + var loadMetadataId = allCataloguesUsedByLoadMetadataNode.LoadMetadata.ID; + var linkedCatalogueIDs = AllLoadMetadataLinkage.Where(link => link.LoadMetadataID == loadMetadataId).Select(static link => link.CatalogueID); + var usedCatalogues = linkedCatalogueIDs.Select(catalogueId => AllCatalogues.FirstOrDefault(c => c.ID == catalogueId)).Where(static foundCatalogue => foundCatalogue is not null).ToList(); + allCataloguesUsedByLoadMetadataNode.UsedCatalogues = usedCatalogues; + var childObjects = usedCatalogues.Select(foundCatalogue => new CatalogueUsedByLoadMetadataNode(allCataloguesUsedByLoadMetadataNode.LoadMetadata, foundCatalogue)).Cast().ToHashSet(); + + AddToDictionaries(childObjects, descendancy); + } + + #endregion + + protected void AddChildren(Catalogue c, DescendancyList descendancy) + { + var childObjects = new List(); + + var catalogueAggregates = AllAggregateConfigurations.Where(a => a.Catalogue_ID == c.ID).ToArray(); + var cohortAggregates = catalogueAggregates.Where(a => a.IsCohortIdentificationAggregate).ToArray(); + var regularAggregates = catalogueAggregates.Except(cohortAggregates).ToArray(); + + //get all the CatalogueItems for this Catalogue (TryGet because Catalogue may not have any items + var cis = _catalogueToCatalogueItems.TryGetValue(c.ID, out var result) + ? result.ToArray() + : Array.Empty(); + + //tell the CatalogueItems that we are are their parent + foreach (var ci in cis) + ci.InjectKnown(c); + + // core includes project specific which basically means the same thing + var core = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Core || + ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.ProjectSpecific) + , ExtractionCategory.Core); + + c.InjectKnown(cis); + + var deprecated = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Deprecated), + ExtractionCategory.Deprecated); + var special = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.SpecialApprovalRequired), + ExtractionCategory.SpecialApprovalRequired); + var intern = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Internal), + ExtractionCategory.Internal); + var supplemental = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Supplemental), + ExtractionCategory.Supplemental); + var notExtractable = new CatalogueItemsNode(c, cis.Where(ci => ci.ExtractionInformation == null), null); + + AddChildren(core, descendancy.Add(core)); + childObjects.Add(core); + + foreach (var optional in new[] { deprecated, special, intern, supplemental, notExtractable }) + if (optional.CatalogueItems.Any()) + { + AddChildren(optional, descendancy.Add(optional)); + childObjects.Add(optional); + } + + //do we have any foreign key fields into this lookup table + var lookups = AllLookups.Where(l => c.CatalogueItems.Any(ci => ci.ColumnInfo_ID == l.ForeignKey_ID)).ToArray(); + + var docs = AllSupportingDocuments.Where(d => d.Catalogue_ID == c.ID).ToArray(); + var sql = AllSupportingSQL.Where(d => d.Catalogue_ID == c.ID).ToArray(); + + //if there are supporting documents or supporting sql files then add documentation node + if (docs.Any() || sql.Any()) + { + var documentationNode = new DocumentationNode(c, docs, sql); + + //add the documentations node + childObjects.Add(documentationNode); + + //record the children + AddToDictionaries(new HashSet(docs.Cast().Union(sql)), descendancy.Add(documentationNode)); + } + + if (lookups.Any()) + { + var lookupsNode = new CatalogueLookupsNode(c, lookups); + //add the documentations node + childObjects.Add(lookupsNode); + + + //record the children + AddToDictionaries(new HashSet(lookups.Select(l => new CatalogueLookupUsageNode(c, l))), + descendancy.Add(lookupsNode)); + } + + if (regularAggregates.Any()) + { + var aggregatesNode = new AggregatesNode(c, regularAggregates); + childObjects.Add(aggregatesNode); + + var nodeDescendancy = descendancy.Add(aggregatesNode); + AddToDictionaries(new HashSet(regularAggregates), nodeDescendancy); + + foreach (var regularAggregate in regularAggregates) + AddChildren(regularAggregate, nodeDescendancy.Add(regularAggregate)); + } + + //finalise + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + private void InjectCatalogueItems() + { + foreach (var ci in AllCatalogueItems) + if (_extractionInformationsByCatalogueItem.TryGetValue(ci.ID, out var ei)) + ci.InjectKnown(ei); + else + ci.InjectKnown((ExtractionInformation)null); + } + + private void AddChildren(CatalogueItemsNode node, DescendancyList descendancyList) + { + AddToDictionaries(new HashSet(node.CatalogueItems), descendancyList); + + foreach (var ci in node.CatalogueItems) + AddChildren(ci, descendancyList.Add(ci)); + } + + private void AddChildren(AggregateConfiguration aggregateConfiguration, DescendancyList descendancy) + { + var childrenObjects = new HashSet(); + + var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(aggregateConfiguration)).Cast() + .ToArray(); + + foreach (var p in parameters) + childrenObjects.Add(p); + + // show the dimensions in the tree + foreach (var dim in aggregateConfiguration.AggregateDimensions) childrenObjects.Add(dim); + + // show the axis (if any) in the tree. If there are multiple axis in this tree then that is bad but maybe the user can delete one of them to fix the situation + foreach (var axis in AllAggregateContinuousDateAxis.Where(a => + aggregateConfiguration.AggregateDimensions.Any(d => d.ID == a.AggregateDimension_ID))) + childrenObjects.Add(axis); + + //we can step into this twice, once via Catalogue children and once via CohortIdentificationConfiguration children + //if we get in via Catalogue children then descendancy will be Ignore=true we don't end up emphasising into CatalogueCollectionUI when + //really user wants to see it in CohortIdentificationCollectionUI + if (aggregateConfiguration.RootFilterContainer_ID != null) + { + var container = AllAggregateContainersDictionary[(int)aggregateConfiguration.RootFilterContainer_ID]; + + AddChildren(container, descendancy.Add(container)); + childrenObjects.Add(container); + } + + AddToDictionaries(childrenObjects, descendancy); + } + + private void AddChildren(AggregateFilterContainer container, DescendancyList descendancy) + { + var childrenObjects = new List(); + + var subcontainers = _aggregateFilterManager.GetSubContainers(container); + var filters = _aggregateFilterManager.GetFilters(container); + + foreach (AggregateFilterContainer subcontainer in subcontainers) + { + //one of our children is this subcontainer + childrenObjects.Add(subcontainer); + + //but also document its children + AddChildren(subcontainer, descendancy.Add(subcontainer)); + } + + //also add the filters for the container + foreach (var f in filters) + { + // for filters add the parameters under them + AddChildren((AggregateFilter)f, descendancy.Add(f)); + childrenObjects.Add(f); + } + + //add our children to the dictionary + AddToDictionaries(new HashSet(childrenObjects), descendancy); + } + + private void AddChildren(AggregateFilter f, DescendancyList descendancy) + { + AddToDictionaries(new HashSet(AllAggregateFilterParameters.Where(p => p.AggregateFilter_ID == f.ID)), + descendancy); + } + + private void AddChildren(CatalogueItem ci, DescendancyList descendancy) + { + var childObjects = new List(); + + var ei = ci.ExtractionInformation; + if (ei != null) + { + childObjects.Add(ei); + AddChildren(ei, descendancy.Add(ei)); + } + else + { + ci.InjectKnown( + (ExtractionInformation)null); // we know the CatalogueItem has no ExtractionInformation child because it's not in the dictionary + } + + if (ci.ColumnInfo_ID.HasValue && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) + childObjects.Add(new LinkedColumnInfoNode(ci, col)); + + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + private void AddChildren(ExtractionInformation extractionInformation, DescendancyList descendancy) + { + var children = new HashSet(); + + foreach (var filter in AllCatalogueFilters.Where(f => f.ExtractionInformation_ID == extractionInformation.ID)) + { + //add the filter as a child of the + children.Add(filter); + AddChildren(filter, descendancy.Add(filter)); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(ExtractionFilter filter, DescendancyList descendancy) + { + var children = new HashSet(); + var parameters = AllCatalogueParameters.Where(p => p.ExtractionFilter_ID == filter.ID).ToArray(); + var parameterSets = AllCatalogueValueSets.Where(vs => vs.ExtractionFilter_ID == filter.ID).ToArray(); + + filter.InjectKnown(parameterSets); + + foreach (var p in parameters) + children.Add(p); + + foreach (var set in parameterSets) + { + children.Add(set); + AddChildren(set, descendancy.Add(set), parameters); + } + + if (children.Any()) + AddToDictionaries(children, descendancy); + } + + private void AddChildren(ExtractionFilterParameterSet set, DescendancyList descendancy, + ExtractionFilterParameter[] filterParameters) + { + var children = new HashSet(); + + foreach (var setValue in AllCatalogueValueSetValues.Where(v => v.ExtractionFilterParameterSet_ID == set.ID)) + { + setValue.InjectKnown(filterParameters.SingleOrDefault(p => p.ID == setValue.ExtractionFilterParameter_ID)); + children.Add(setValue); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(CohortIdentificationConfiguration cic, DescendancyList descendancy) + { + var children = new HashSet(); + + //it has an associated query cache + if (cic.QueryCachingServer_ID != null) + children.Add(new QueryCacheUsedByCohortIdentificationNode(cic, + AllExternalServers.Single(s => s.ID == cic.QueryCachingServer_ID))); + + var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(cic)).Cast().ToArray(); + foreach (var p in parameters) children.Add(p); + + //if it has a root container + if (cic.RootCohortAggregateContainer_ID != null) + { + var container = AllCohortAggregateContainers.Single(c => c.ID == cic.RootCohortAggregateContainer_ID); + AddChildren(container, descendancy.Add(container).SetBetterRouteExists()); + children.Add(container); + } + + //get the patient index tables + var joinableNode = new JoinableCollectionNode(cic, + AllJoinables.Where(j => j.CohortIdentificationConfiguration_ID == cic.ID).ToArray()); + AddChildren(joinableNode, descendancy.Add(joinableNode).SetBetterRouteExists()); + children.Add(joinableNode); + + AddToDictionaries(children, descendancy.SetBetterRouteExists()); + } + + private void AddChildren(JoinableCollectionNode joinablesNode, DescendancyList descendancy) + { + var children = new HashSet(); + + foreach (var joinable in joinablesNode.Joinables) + try + { + var agg = AllAggregateConfigurations.Single(ac => ac.ID == joinable.AggregateConfiguration_ID); + ForceAggregateNaming(agg, descendancy); + children.Add(agg); + + //it's no longer an orphan because it's in a known cic (as a patient index table) + OrphanAggregateConfigurations.Remove(agg); + + AddChildren(agg, descendancy.Add(agg)); + } + catch (Exception e) + { + throw new Exception( + $"JoinableCohortAggregateConfiguration (patient index table) object (ID={joinable.ID}) references AggregateConfiguration_ID {joinable.AggregateConfiguration_ID} but that AggregateConfiguration was not found", + e); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(CohortAggregateContainer container, DescendancyList descendancy) + { + //get subcontainers + var subcontainers = _cohortContainerManager.GetChildren(container).OfType().ToList(); + + //if there are subcontainers + foreach (var subcontainer in subcontainers) + AddChildren(subcontainer, descendancy.Add(subcontainer)); + + //get our configurations + var configurations = _cohortContainerManager.GetChildren(container).OfType().ToList(); + + //record the configurations children including full descendancy + foreach (var configuration in configurations) + { + ForceAggregateNaming(configuration, descendancy); + AddChildren(configuration, descendancy.Add(configuration)); + + //it's no longer an orphan because it's in a known cic + OrphanAggregateConfigurations.Remove(configuration); + } + + //all our children (containers and aggregates) + //children are all aggregates and containers at the current hierarchy level in order + var children = subcontainers.Union(configurations.Cast()).OrderBy(o => o.Order).ToList(); + + AddToDictionaries(new HashSet(children), descendancy); + } + + private void ForceAggregateNaming(AggregateConfiguration configuration, DescendancyList descendancy) + { + //configuration has the wrong name + if (!configuration.IsCohortIdentificationAggregate) + { + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( + $"Had to fix naming of configuration '{configuration}' because it didn't start with correct cic prefix", + CheckResult.Warning)); + descendancy.Parents.OfType().Single() + .EnsureNamingConvention(configuration); + configuration.SaveToDatabase(); + } + } + + private void AddChildren(TableInfoServerNode serverNode, DescendancyList descendancy) + { + //add empty hashset + var children = new HashSet(); + + var databases = + serverNode.Tables.GroupBy( + k => k.Database ?? TableInfoDatabaseNode.NullDatabaseNode, StringComparer.CurrentCultureIgnoreCase) + .Select(g => new TableInfoDatabaseNode(g.Key, serverNode, g)); + + foreach (var db in databases) + { + children.Add(db); + AddChildren(db, descendancy.Add(db)); + } + + //now we have recorded all the children add them with descendancy + AddToDictionaries(children, descendancy); + } + + private void AddChildren(TableInfoDatabaseNode dbNode, DescendancyList descendancy) + { + //add empty hashset + var children = new HashSet(); + + foreach (var t in dbNode.Tables) + { + //record the children of the table infos (mostly column infos) + children.Add(t); + + //the all servers node=>the TableInfoServerNode => the t + AddChildren(t, descendancy.Add(t)); + } + + //now we have recorded all the children add them with descendancy + AddToDictionaries(children, descendancy); + } + + private void AddChildren(TableInfo tableInfo, DescendancyList descendancy) + { + //add empty hashset + var children = new HashSet(); + + //if the table has an identifier dump listed + if (tableInfo.IdentifierDumpServer_ID != null) + { + //if there is a dump (e.g. for dilution and dumping - not appearing in the live table) + var server = AllExternalServers.Single(s => s.ID == tableInfo.IdentifierDumpServer_ID.Value); + + children.Add(new IdentifierDumpServerUsageNode(tableInfo, server)); + } + + //get the discarded columns in this table + var discardedCols = new HashSet(AllPreLoadDiscardedColumns.Where(c => c.TableInfo_ID == tableInfo.ID)); + + //tell the column who their parent is so they don't need to look up the database + foreach (PreLoadDiscardedColumn discardedCol in discardedCols) + discardedCol.InjectKnown(tableInfo); + + //if there are discarded columns + if (discardedCols.Any()) + { + var identifierDumpNode = new PreLoadDiscardedColumnsNode(tableInfo); + + //record that the usage is a child of TableInfo + children.Add(identifierDumpNode); + + //record that the discarded columns are children of identifier dump usage node + AddToDictionaries(discardedCols, descendancy.Add(identifierDumpNode)); + } + + //if it is a table valued function + if (tableInfo.IsTableValuedFunction) + { + //that has parameters + var parameters = tableInfo.GetAllParameters(); + + foreach (var p in parameters) children.Add(p); + } + + //next add the column infos + if (TableInfosToColumnInfos.TryGetValue(tableInfo.ID, out var result)) + foreach (var c in result) + { + children.Add(c); + c.InjectKnown(tableInfo); + AddChildren(c, descendancy.Add(c).SetBetterRouteExists()); + } + + //finally add any credentials objects + if (AllDataAccessCredentialUsages.TryGetValue(tableInfo, out var nodes)) + foreach (var node in nodes) + children.Add(node); + + //now we have recorded all the children add them with descendancy via the TableInfo descendancy + AddToDictionaries(children, descendancy); + } + + private void AddChildren(ColumnInfo columnInfo, DescendancyList descendancy) + { + var lookups = AllLookups.Where(l => l.Description_ID == columnInfo.ID).ToArray(); + var joinInfos = AllJoinInfos.Where(j => j.PrimaryKey_ID == columnInfo.ID); + + var children = new HashSet(); + + foreach (var l in lookups) + children.Add(l); + + foreach (var j in joinInfos) + children.Add(j); + + if (children.Any()) + AddToDictionaries(children, descendancy); + } + + + protected void AddToDictionaries(HashSet children, DescendancyList list) + { + if (list.IsEmpty) + throw new ArgumentException("DescendancyList cannot be empty", nameof(list)); + + //document that the last parent has these as children + var parent = list.Last(); + + _childDictionary.AddOrUpdate(parent, + children, (p, s) => children); + + //now document the entire parent order to reach each child object i.e. 'Root=>Grandparent=>Parent' is how you get to 'Child' + foreach (var o in children) + _descendancyDictionary.AddOrUpdate(o, list, (k, v) => HandleDescendancyCollision(k, v, list)); + + + foreach (var masquerader in children.OfType()) + { + var key = masquerader.MasqueradingAs(); + + if (!AllMasqueraders.ContainsKey(key)) + AllMasqueraders.AddOrUpdate(key, new HashSet(), (o, set) => set); + + lock (AllMasqueraders) + { + AllMasqueraders[key].Add(masquerader); + } + } + } + + private static DescendancyList HandleDescendancyCollision(object key, DescendancyList oldRoute, + DescendancyList newRoute) + { + //if the new route is the best best + if (newRoute.NewBestRoute && !oldRoute.NewBestRoute) + return newRoute; + + // If the new one is marked BetterRouteExists just throw away the new one + return newRoute.BetterRouteExists ? oldRoute : newRoute; + // If in doubt use the newest one + } + + private HashSet GetAllObjects() + { + //anything which has children or is a child of someone else (distinct because HashSet) + return new HashSet(_childDictionary.SelectMany(kvp => kvp.Value).Union(_childDictionary.Keys)); + } + + public virtual object[] GetChildren(object model) + { + lock (WriteLock) + { + ; //if we have a record of any children in the child dictionary for the parent model object + if (_childDictionary.TryGetValue(model, out var cached)) + return cached.OrderBy(static o => o.ToString()).ToArray(); + + return model switch + { + //if they want the children of a Pipeline (which we don't track) just serve the components + Pipeline p => p.PipelineComponents.ToArray(), + //if they want the children of a PipelineComponent (which we don't track) just serve the arguments + PipelineComponent pc => pc.PipelineComponentArguments.ToArray(), + _ => Array.Empty() + }; + } + } + + public IEnumerable GetAllObjects(Type type, bool unwrapMasqueraders) + { + lock (WriteLock) + { + //things that are a match on Type but not IMasqueradeAs + var exactMatches = GetAllSearchables().Keys.Where(t => t is not IMasqueradeAs).Where(type.IsInstanceOfType); + + //Union the unwrapped masqueraders + return unwrapMasqueraders + ? exactMatches.Union( + AllMasqueraders + .Select(kvp => kvp.Key) + .OfType() + .Where(type.IsInstanceOfType)) + .Distinct() + : exactMatches; + } + } + + public DescendancyList GetDescendancyListIfAnyFor(object model) + { + lock (WriteLock) + { + return _descendancyDictionary.GetValueOrDefault(model); + } + } + + + public object GetRootObjectOrSelf(object objectToEmphasise) + { + lock (WriteLock) + { + var descendancy = GetDescendancyListIfAnyFor(objectToEmphasise); + + return descendancy != null && descendancy.Parents.Any() ? descendancy.Parents[0] : objectToEmphasise; + } + } + + + public virtual Dictionary GetAllSearchables() + { + lock (WriteLock) + { + var toReturn = new Dictionary(); + + foreach (var kvp in _descendancyDictionary.Where(kvp => kvp.Key is IMapsDirectlyToDatabaseTable)) + toReturn.Add((IMapsDirectlyToDatabaseTable)kvp.Key, kvp.Value); + + return toReturn; + } + } + + public IEnumerable GetAllChildrenRecursively(object o) + { + lock (WriteLock) + { + var toReturn = new List(); + + foreach (var child in GetChildren(o)) + { + toReturn.Add(child); + toReturn.AddRange(GetAllChildrenRecursively(child)); + } + + return toReturn; + } + } + + /// + /// Asks all plugins to provide the child objects for every object we have found so far. This method is recursive, call it with null the first time to use all objects. It will then + /// call itself with all the new objects that were sent back by the plugin (so that new objects found can still have children). + /// + /// + protected void GetPluginChildren(HashSet objectsToAskAbout = null) + { + lock (WriteLock) + { + var newObjectsFound = new HashSet(); + + var sw = new Stopwatch(); + + var providers = _pluginChildProviders.Except(_blockedPlugins).ToArray(); + + //for every object found so far + if (providers.Any()) + foreach (var o in objectsToAskAbout ?? GetAllObjects()) + //for every plugin loaded (that is not forbidlisted) + foreach (var plugin in providers) + //ask about the children + try + { + sw.Restart(); + //otherwise ask plugin what its children are + var pluginChildren = plugin.GetChildren(o); + + //if the plugin takes too long to respond we need to stop + if (sw.ElapsedMilliseconds > 1000) + { + _blockedPlugins.Add(plugin); + throw new Exception( + $"Plugin '{plugin}' was forbidlisted for taking too long to respond to GetChildren(o) where o was a '{o.GetType().Name}' ('{o}')"); + } + + //it has children + if (pluginChildren != null && pluginChildren.Any()) + { + //get the descendancy of the parent + var parentDescendancy = GetDescendancyListIfAnyFor(o); + var newDescendancy = parentDescendancy == null + ? new DescendancyList(new[] { o }) + : //if the parent is a root level object start a new descendancy list from it + parentDescendancy + .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one + newDescendancy = + parentDescendancy + .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one + + //record that + foreach (var pluginChild in pluginChildren) + { + //if the parent didn't have any children before + if (!_childDictionary.ContainsKey(o)) + _childDictionary.AddOrUpdate(o, new HashSet(), + (o1, set) => set); //it does now + + + //add us to the parent objects child collection + _childDictionary[o].Add(pluginChild); + + //add to the child collection of the parent object kvp.Key + _descendancyDictionary.AddOrUpdate(pluginChild, newDescendancy, + (s, e) => newDescendancy); + + //we have found a new object so we must ask other plugins about it (chances are a plugin will have a whole tree of sub objects) + newObjectsFound.Add(pluginChild); + } + } + } + catch (Exception e) + { + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail, e)); + } + + if (newObjectsFound.Any()) + GetPluginChildren(newObjectsFound); + } + } + + public IEnumerable GetMasqueradersOf(object o) + { + lock (WriteLock) + { + return AllMasqueraders.TryGetValue(o, out var result) ? result : Array.Empty(); + } + } + + protected T[] GetAllObjects(IRepository repository) where T : IMapsDirectlyToDatabaseTable + { + lock (WriteLock) + { + return repository.GetAllObjects(); + } + } + + + protected void AddToReturnSearchablesWithNoDecendancy( + Dictionary toReturn, + IEnumerable toAdd) + { + lock (WriteLock) + { + foreach (var m in toAdd) + toReturn.Add(m, null); + } + } + + public virtual void UpdateTo(ICoreChildProvider other) + { + ArgumentNullException.ThrowIfNull(other); + + if (other is not CatalogueChildProvider otherCat) + throw new NotSupportedException( + $"Did not know how to UpdateTo ICoreChildProvider of type {other.GetType().Name}"); + + AllLoadMetadatas = otherCat.AllLoadMetadatas; + AllProcessTasks = otherCat.AllProcessTasks; + AllProcessTasksArguments = otherCat.AllProcessTasksArguments; + AllLoadProgresses = otherCat.AllLoadProgresses; + AllCacheProgresses = otherCat.AllCacheProgresses; + AllPermissionWindows = otherCat.AllPermissionWindows; + AllCatalogues = otherCat.AllCatalogues; + AllCataloguesDictionary = otherCat.AllCataloguesDictionary; + AllSupportingDocuments = otherCat.AllSupportingDocuments; + AllSupportingSQL = otherCat.AllSupportingSQL; + _childDictionary = otherCat._childDictionary; + _descendancyDictionary = otherCat._descendancyDictionary; + _catalogueToCatalogueItems = otherCat._catalogueToCatalogueItems; + AllCatalogueItemsDictionary = otherCat.AllCatalogueItemsDictionary; + _allColumnInfos = otherCat._allColumnInfos; + AllAggregateConfigurations = otherCat.AllAggregateConfigurations; + AllAggregateDimensions = otherCat.AllAggregateDimensions; + AllAggregateContinuousDateAxis = otherCat.AllAggregateContinuousDateAxis; + AllRDMPRemotesNode = otherCat.AllRDMPRemotesNode; + AllRemoteRDMPs = otherCat.AllRemoteRDMPs; + AllDashboardsNode = otherCat.AllDashboardsNode; + AllDashboards = otherCat.AllDashboards; + AllObjectSharingNode = otherCat.AllObjectSharingNode; + AllImports = otherCat.AllImports; + AllExports = otherCat.AllExports; + AllStandardRegexesNode = otherCat.AllStandardRegexesNode; + AllPipelinesNode = otherCat.AllPipelinesNode; + OtherPipelinesNode = otherCat.OtherPipelinesNode; + AllPipelines = otherCat.AllPipelines; + AllPipelineComponents = otherCat.AllPipelineComponents; + AllPipelineComponentsArguments = otherCat.AllPipelineComponentsArguments; + AllStandardRegexes = otherCat.AllStandardRegexes; + AllANOTablesNode = otherCat.AllANOTablesNode; + AllANOTables = otherCat.AllANOTables; + AllExternalServers = otherCat.AllExternalServers; + AllServers = otherCat.AllServers; + AllTableInfos = otherCat.AllTableInfos; + AllDataAccessCredentialsNode = otherCat.AllDataAccessCredentialsNode; + AllExternalServersNode = otherCat.AllExternalServersNode; + AllServersNode = otherCat.AllServersNode; + AllDataAccessCredentials = otherCat.AllDataAccessCredentials; + AllDataAccessCredentialUsages = otherCat.AllDataAccessCredentialUsages; + TableInfosToColumnInfos = otherCat.TableInfosToColumnInfos; + AllColumnInfos = otherCat.AllColumnInfos; + AllPreLoadDiscardedColumns = otherCat.AllPreLoadDiscardedColumns; + AllLookups = otherCat.AllLookups; + AllJoinInfos = otherCat.AllJoinInfos; + AllAnyTableParameters = otherCat.AllAnyTableParameters; + AllMasqueraders = otherCat.AllMasqueraders; + AllExtractionInformationsDictionary = otherCat.AllExtractionInformationsDictionary; + _pluginChildProviders = otherCat._pluginChildProviders; + AllPermissionWindowsNode = otherCat.AllPermissionWindowsNode; + LoadMetadataRootFolder = otherCat.LoadMetadataRootFolder; + CatalogueRootFolder = otherCat.CatalogueRootFolder; + CohortIdentificationConfigurationRootFolder = otherCat.CohortIdentificationConfigurationRootFolder; + AllConnectionStringKeywordsNode = otherCat.AllConnectionStringKeywordsNode; + AllConnectionStringKeywords = otherCat.AllConnectionStringKeywords; + AllAggregateContainersDictionary = otherCat.AllAggregateContainersDictionary; + AllAggregateFilters = otherCat.AllAggregateFilters; + AllAggregateFilterParameters = otherCat.AllAggregateFilterParameters; + AllCohortIdentificationConfigurations = otherCat.AllCohortIdentificationConfigurations; + AllCohortAggregateContainers = otherCat.AllCohortAggregateContainers; + AllJoinables = otherCat.AllJoinables; + AllJoinUses = otherCat.AllJoinUses; + AllGovernanceNode = otherCat.AllGovernanceNode; + AllGovernancePeriods = otherCat.AllGovernancePeriods; + AllGovernanceDocuments = otherCat.AllGovernanceDocuments; + GovernanceCoverage = otherCat.GovernanceCoverage; + AllJoinableCohortAggregateConfigurationUse = otherCat.AllJoinableCohortAggregateConfigurationUse; + AllPluginsNode = otherCat.AllPluginsNode; + PipelineUseCases = otherCat.PipelineUseCases; + OrphanAggregateConfigurationsNode = otherCat.OrphanAggregateConfigurationsNode; + TemplateAggregateConfigurationsNode = otherCat.TemplateAggregateConfigurationsNode; + AllCatalogueParameters = otherCat.AllCatalogueParameters; + AllCatalogueValueSets = otherCat.AllCatalogueValueSets; + AllCatalogueValueSetValues = otherCat.AllCatalogueValueSetValues; + OrphanAggregateConfigurations = otherCat.OrphanAggregateConfigurations; + AllTemplateCohortIdentificationConfigurationsNode = other.AllTemplateCohortIdentificationConfigurationsNode; + + } + + public virtual bool SelectiveRefresh(IMapsDirectlyToDatabaseTable databaseEntity) + { + ProgressStopwatch.Restart(); + + return databaseEntity switch + { + AggregateFilterParameter afp => SelectiveRefresh(afp.AggregateFilter), + AggregateFilter af => SelectiveRefresh(af), + AggregateFilterContainer afc => SelectiveRefresh(afc), + CohortAggregateContainer cac => SelectiveRefresh(cac), + ExtractionInformation ei => SelectiveRefresh(ei), + CatalogueItem ci => SelectiveRefresh(ci), + _ => false + }; + } + + + public bool SelectiveRefresh(CatalogueItem ci) + { + var descendancy = GetDescendancyListIfAnyFor(ci.Catalogue); + if (descendancy == null) return false; + + FetchCatalogueItems(); + FetchExtractionInformations(); + AddChildren(ci.Catalogue, descendancy.Add(ci.Catalogue)); + return true; + } + + public bool SelectiveRefresh(ExtractionInformation ei) + { + var descendancy = GetDescendancyListIfAnyFor(ei); + var cata = descendancy?.Parents.OfType().LastOrDefault() ?? ei.CatalogueItem.Catalogue; + + if (cata == null) + return false; + + var cataDescendancy = GetDescendancyListIfAnyFor(cata); + + if (cataDescendancy == null) + return false; + + FetchCatalogueItems(); + + foreach (var ci in AllCatalogueItems.Where(ci => ci.ID == ei.CatalogueItem_ID)) ci.ClearAllInjections(); + + // property changes or deleting the ExtractionInformation + FetchExtractionInformations(); + + // refresh the Catalogue + AddChildren(cata, cataDescendancy.Add(cata)); + return true; + } + + public bool SelectiveRefresh(CohortAggregateContainer container) + { + var parentContainer = container.GetParentContainerIfAny(); + if (parentContainer != null) + { + var descendancy = GetDescendancyListIfAnyFor(parentContainer); + + if (descendancy != null) + { + BuildAggregateConfigurations(); + + BuildCohortCohortAggregateContainers(); + AddChildren(parentContainer, descendancy.Add(parentContainer)); + return true; + } + } + + var cic = container.GetCohortIdentificationConfiguration(); + + if (cic != null) + { + var descendancy = GetDescendancyListIfAnyFor(cic); + + if (descendancy != null) + { + BuildAggregateConfigurations(); + BuildCohortCohortAggregateContainers(); + AddChildren(cic, descendancy.Add(cic)); + return true; + } + } + + return false; + } + + public bool SelectiveRefresh(AggregateFilter f) + { + var knownContainer = GetDescendancyListIfAnyFor(f.FilterContainer); + if (knownContainer == null) return false; + + BuildAggregateFilterContainers(); + AddChildren((AggregateFilterContainer)f.FilterContainer, knownContainer.Add(f.FilterContainer)); + return true; + } - public bool SelectiveRefresh(AggregateFilterContainer container) - { - var aggregate = container.GetAggregate(); + public bool SelectiveRefresh(AggregateFilterContainer container) + { + var aggregate = container.GetAggregate(); - if (aggregate == null) return false; + if (aggregate == null) return false; - var descendancy = GetDescendancyListIfAnyFor(aggregate); + var descendancy = GetDescendancyListIfAnyFor(aggregate); - if (descendancy == null) return false; + if (descendancy == null) return false; - // update just in case we became a root filter for someone - aggregate.RevertToDatabaseState(); + // update just in case we became a root filter for someone + aggregate.RevertToDatabaseState(); - BuildAggregateFilterContainers(); + BuildAggregateFilterContainers(); - AddChildren(aggregate, descendancy.Add(aggregate)); - return true; - } + AddChildren(aggregate, descendancy.Add(aggregate)); + return true; + } } \ No newline at end of file From ab3489704a9373baecde2dd54f8257467a8740e9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 09:42:18 +0000 Subject: [PATCH 118/142] update readonly check and docs --- CHANGELOG.md | 1 + ...dCreateCohortIdentificationConfigurationTemplate.cs | 1 + ...mandUseTemplateCohortIdentificationConfiguration.cs | 1 + ...eConfigurationToCohortIdentificationSetContainer.cs | 2 +- ...ndAddCatalogueToCohortIdentificationSetContainer.cs | 2 +- ...ConvertAggregateConfigurationToPatientIndexTable.cs | 2 +- .../AtomicCommands/ExecuteCommandDelete.cs | 2 +- .../AtomicCommands/ExecuteCommandDisableOrEnable.cs | 2 +- ...teCommandImportCohortIdentificationConfiguration.cs | 2 +- ...ndexTableIntoRegularCohortIdentificationSetAgain.cs | 2 +- .../ExecuteCommandMoveAggregateIntoContainer.cs | 2 +- ...mandMoveCohortAggregateContainerIntoSubContainer.cs | 2 +- .../ExecuteCommandMoveContainerIntoContainer.cs | 2 +- .../ExecuteCommandMoveFilterIntoContainer.cs | 2 +- .../AtomicCommands/ExecuteCommandRename.cs | 2 +- .../ExecuteCommandSetContainerOperation.cs | 2 +- .../ExecutecommandAddCohortSubContainer.cs | 2 +- Rdmp.Core/CommandExecution/BasicCommandExecution.cs | 2 +- .../Data/Aggregation/AggregateConfiguration.cs | 4 ++-- .../Data/Aggregation/AggregateFilterContainer.cs | 5 +++-- .../Curation/Data/Cohort/CohortAggregateContainer.cs | 4 ++-- .../Data/Cohort/CohortIdentificationConfiguration.cs | 10 +++++++++- Rdmp.Core/Curation/Data/ConcreteContainer.cs | 2 +- Rdmp.Core/Curation/Data/ConcreteFilter.cs | 4 ++-- Rdmp.Core/Curation/Data/IMightBeReadOnly.cs | 2 +- .../SpontaneouslyInventedFilterContainer.cs | 2 +- .../FilterImporting/ParameterCollectionUIOptions.cs | 2 +- Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs | 2 +- Rdmp.Core/DataExport/Data/FilterContainer.cs | 4 ++-- ...jectCohortIdentificationConfigurationAssociation.cs | 4 ++-- Rdmp.Core/DataExport/Data/SelectedDataSets.cs | 2 +- .../CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql | 2 +- .../AtomicCommands/ExecuteCommandReOrderAggregate.cs | 2 +- .../ExecuteCommandReOrderAggregateContainer.cs | 2 +- .../RDMPSingleDatabaseObjectControl.cs | 2 +- 35 files changed, 50 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2052c0f96b..971a83ec2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.0.4] - Unreleased - Fix bug with duplicate searchables +- Add the ability to Template Cohort Identification Configurations (see [Documentation\cohorts\CohortIdentificationConfigurationTemplates.md]) ## [9.0.3] - 2025-11-03 - Improve checking for default pipelines diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs index eca2009d1f..4eef191774 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs @@ -63,6 +63,7 @@ public override void Execute() } } Publish(clone); + Emphasise(clone); } public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs index 186dd82d4c..94f6f13284 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs @@ -68,6 +68,7 @@ public override void Execute() cmd.Execute(); } Publish(clone); + Emphasise(clone); } public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer.cs index a589b93d46..4bcaa678ac 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer.cs @@ -44,7 +44,7 @@ private ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContaine { _targetCohortAggregateContainer = targetCohortAggregateContainer; - if (targetCohortAggregateContainer.ShouldBeReadOnly(out var reason)) + if (targetCohortAggregateContainer.ShouldBeReadOnly(this.GetType().Name,out var reason)) SetImpossible(reason); UseTripleDotSuffix = true; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs index 3adaf4efbd..114cd2974e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs @@ -51,7 +51,7 @@ public ExecuteCommandAddCatalogueToCohortIdentificationSetContainer(IBasicActiva _targetCohortAggregateContainer = targetCohortAggregateContainer; - if (targetCohortAggregateContainer.ShouldBeReadOnly(out var reason)) + if (targetCohortAggregateContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); if (catalogue != null) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandConvertAggregateConfigurationToPatientIndexTable.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandConvertAggregateConfigurationToPatientIndexTable.cs index ef7edb2cf8..eb9758c40a 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandConvertAggregateConfigurationToPatientIndexTable.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandConvertAggregateConfigurationToPatientIndexTable.cs @@ -40,7 +40,7 @@ public ExecuteCommandConvertAggregateConfigurationToPatientIndexTable(IBasicActi SetImpossible( $"Aggregate '{_sourceAggregateConfigurationCombineable.Aggregate}' belongs to a different Cohort Identification Configuration"); - if (cic != null && cic.ShouldBeReadOnly(out var reason)) + if (cic != null && cic.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs index ef4a191c58..38ba6a9925 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs @@ -50,7 +50,7 @@ public ExecuteCommandDelete(IBasicActivateItems activator, SetImpossible("Cannot delete root containers"); var reason = ""; - if (_deletables.Any(d => d is IMightBeReadOnly ro && ro.ShouldBeReadOnly(out reason))) + if (_deletables.Any(d => d is IMightBeReadOnly ro && ro.ShouldBeReadOnly(this.GetType().Name, out reason))) SetImpossible(reason); Weight = 50.4f; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs index d2008fff56..cb94ded66b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs @@ -62,7 +62,7 @@ private void UpdateViabilityForTarget(IDisableable target) } } - if (target is IMightBeReadOnly ro && ro.ShouldBeReadOnly(out var reason)) + if (target is IMightBeReadOnly ro && ro.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportCohortIdentificationConfiguration.cs index 45d145e372..cbeac6b2d8 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportCohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportCohortIdentificationConfiguration.cs @@ -36,7 +36,7 @@ public ExecuteCommandImportCohortIdentificationConfiguration(IBasicActivateItems return; } - if (intoContainer.ShouldBeReadOnly(out var reason)) SetImpossible(reason); + if (intoContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); // if we don't know what to import yet then we should have the // 'more choices to come' suffix diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakePatientIndexTableIntoRegularCohortIdentificationSetAgain.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakePatientIndexTableIntoRegularCohortIdentificationSetAgain.cs index a2431b0556..7323fd4bfe 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakePatientIndexTableIntoRegularCohortIdentificationSetAgain.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakePatientIndexTableIntoRegularCohortIdentificationSetAgain.cs @@ -39,7 +39,7 @@ public ExecuteCommandMakePatientIndexTableIntoRegularCohortIdentificationSetAgai SetImpossible( $"The following Cohort Set(s) use this PatientIndex table:{string.Join(",", _sourceAggregateCommand.JoinableUsersIfAny.Select(j => j.ToString()))}"); - if (_targetCohortAggregateContainer.ShouldBeReadOnly(out var reason)) + if (_targetCohortAggregateContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveAggregateIntoContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveAggregateIntoContainer.cs index 34dd194502..cc4949587a 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveAggregateIntoContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveAggregateIntoContainer.cs @@ -39,7 +39,7 @@ public ExecuteCommandMoveAggregateIntoContainer(IBasicActivateItems activator, _sourceAggregateCommand.ContainerIfAny.Equals(targetCohortAggregateContainer)) SetImpossible("Aggregate is already in container"); - if (targetCohortAggregateContainer.ShouldBeReadOnly(out var reason)) + if (targetCohortAggregateContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveCohortAggregateContainerIntoSubContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveCohortAggregateContainerIntoSubContainer.cs index 7d122d6c98..f82edf3214 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveCohortAggregateContainerIntoSubContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveCohortAggregateContainerIntoSubContainer.cs @@ -35,7 +35,7 @@ public ExecuteCommandMoveCohortAggregateContainerIntoSubContainer(IBasicActivate if (_sourceCohortAggregateContainer.AggregateContainer.Equals(_targetCohortAggregateContainer)) SetImpossible("Cannot move a container into itself"); - if (_targetCohortAggregateContainer.ShouldBeReadOnly(out var reason)) + if (_targetCohortAggregateContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveContainerIntoContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveContainerIntoContainer.cs index f1d4de1e4c..8b85311d2b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveContainerIntoContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveContainerIntoContainer.cs @@ -30,7 +30,7 @@ public ExecuteCommandMoveContainerIntoContainer(IBasicActivateItems activator, if (containerCombineable.AllSubContainersRecursive.Contains(targetContainer)) SetImpossible("You cannot move a container (AND/OR) into one of its own subcontainers"); - if (targetContainer.ShouldBeReadOnly(out var reason)) + if (targetContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveFilterIntoContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveFilterIntoContainer.cs index 615a96433b..556f8abfa5 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveFilterIntoContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveFilterIntoContainer.cs @@ -30,7 +30,7 @@ public ExecuteCommandMoveFilterIntoContainer(IBasicActivateItems activator, Filt if (!filterCombineable.AllContainersInEntireTreeFromRootDown.Contains(targetContainer)) SetImpossible("Filters can only be moved within their own container tree"); - if (targetContainer.ShouldBeReadOnly(out var reason)) + if (targetContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRename.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRename.cs index 1ea4106247..1c5b391206 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRename.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRename.cs @@ -27,7 +27,7 @@ public ExecuteCommandRename(IBasicActivateItems activator, INamed nameable) : ba case ITableInfo: SetImpossible("TableInfos cannot not be renamed"); break; - case IMightBeReadOnly ro when ro.ShouldBeReadOnly(out var reason): + case IMightBeReadOnly ro when ro.ShouldBeReadOnly(this.GetType().Name, out var reason): SetImpossible(reason); break; } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetContainerOperation.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetContainerOperation.cs index bb8818160a..f141522f7b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetContainerOperation.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetContainerOperation.cs @@ -29,7 +29,7 @@ public ExecuteCommandSetContainerOperation(IBasicActivateItems activator, Cohort _container = container; _operation = operation; - if (container.ShouldBeReadOnly(out var reason)) SetImpossible(reason); + if (container.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); Weight = _operation switch { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecutecommandAddCohortSubContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecutecommandAddCohortSubContainer.cs index 01725cb730..7e1d61675b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecutecommandAddCohortSubContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecutecommandAddCohortSubContainer.cs @@ -22,7 +22,7 @@ public ExecuteCommandAddCohortSubContainer(IBasicActivateItems activator, Cohort Weight = 0.12f; _container = container; - if (container.ShouldBeReadOnly(out var reason)) SetImpossible(reason); + if (container.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } public override Image GetImage(IIconProvider iconProvider) => diff --git a/Rdmp.Core/CommandExecution/BasicCommandExecution.cs b/Rdmp.Core/CommandExecution/BasicCommandExecution.cs index 07de059b51..f7d90f428f 100644 --- a/Rdmp.Core/CommandExecution/BasicCommandExecution.cs +++ b/Rdmp.Core/CommandExecution/BasicCommandExecution.cs @@ -86,7 +86,7 @@ public abstract class BasicCommandExecution : IAtomicCommand protected void SetImpossibleIfReadonly(IMightBeReadOnly m) { - if (m?.ShouldBeReadOnly(out var reason) == true) + if (m?.ShouldBeReadOnly(this.GetType().Name,out var reason) == true) SetImpossible( $"{(m is IContainer ? "Container" : '\'' + m.ToString() + '\'')} is readonly because:{reason}"); } diff --git a/Rdmp.Core/Curation/Data/Aggregation/AggregateConfiguration.cs b/Rdmp.Core/Curation/Data/Aggregation/AggregateConfiguration.cs index 2699e206d4..4ff492e4a7 100644 --- a/Rdmp.Core/Curation/Data/Aggregation/AggregateConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Aggregation/AggregateConfiguration.cs @@ -448,7 +448,7 @@ public override string ToString() => //strip the cic section from the front Regex.Replace(Name, $@"{CohortIdentificationConfiguration.CICPrefix}\d+_?", ""); - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context,out string reason) { var cic = GetCohortIdentificationConfigurationIfAny(); if (cic == null) @@ -457,7 +457,7 @@ public bool ShouldBeReadOnly(out string reason) return false; } - return cic.ShouldBeReadOnly(out reason); + return cic.ShouldBeReadOnly(context,out reason); } /// diff --git a/Rdmp.Core/Curation/Data/Aggregation/AggregateFilterContainer.cs b/Rdmp.Core/Curation/Data/Aggregation/AggregateFilterContainer.cs index d60856b272..cf00bf96bc 100644 --- a/Rdmp.Core/Curation/Data/Aggregation/AggregateFilterContainer.cs +++ b/Rdmp.Core/Curation/Data/Aggregation/AggregateFilterContainer.cs @@ -74,9 +74,10 @@ public override Catalogue GetCatalogueIfAny() /// /// Returns true if the filter container belongs to a parent that is frozen /// + /// /// /// - public override bool ShouldBeReadOnly(out string reason) + public override bool ShouldBeReadOnly(string context, out string reason) { var cic = GetAggregate()?.GetCohortIdentificationConfigurationIfAny(); if (cic == null) @@ -85,7 +86,7 @@ public override bool ShouldBeReadOnly(out string reason) return false; } - return cic.ShouldBeReadOnly(out reason); + return cic.ShouldBeReadOnly(context, out reason); } /// diff --git a/Rdmp.Core/Curation/Data/Cohort/CohortAggregateContainer.cs b/Rdmp.Core/Curation/Data/Cohort/CohortAggregateContainer.cs index a0d7314bec..65ae9c2a77 100644 --- a/Rdmp.Core/Curation/Data/Cohort/CohortAggregateContainer.cs +++ b/Rdmp.Core/Curation/Data/Cohort/CohortAggregateContainer.cs @@ -203,7 +203,7 @@ public override void DeleteInDatabase() /// public override string ToString() => Name; - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context, out string reason) { var cic = GetCohortIdentificationConfiguration(); @@ -213,7 +213,7 @@ public bool ShouldBeReadOnly(out string reason) return false; } - return cic.ShouldBeReadOnly(out reason); + return cic.ShouldBeReadOnly(context,out reason); } /// diff --git a/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs b/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs index e0c80857f8..6982d425f9 100644 --- a/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs @@ -283,8 +283,16 @@ public void CreateRootContainerIfNotExists() /// public override string ToString() => Name; - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context, out string reason) { + if (IsTemplate) + { + if (context == "ExecuteCommandRename") + { + reason = null; + return false; + } + } if (Frozen) { reason = $"{Name} is Frozen"; diff --git a/Rdmp.Core/Curation/Data/ConcreteContainer.cs b/Rdmp.Core/Curation/Data/ConcreteContainer.cs index d93919c48a..bb1d5d5f52 100644 --- a/Rdmp.Core/Curation/Data/ConcreteContainer.cs +++ b/Rdmp.Core/Curation/Data/ConcreteContainer.cs @@ -154,7 +154,7 @@ private List GetAllSubContainersRecursively(IContainer current) } /// - public abstract bool ShouldBeReadOnly(out string reason); + public abstract bool ShouldBeReadOnly(string context, out string reason); public abstract IContainer DeepCloneEntireTreeRecursivelyIncludingFilters(); diff --git a/Rdmp.Core/Curation/Data/ConcreteFilter.cs b/Rdmp.Core/Curation/Data/ConcreteFilter.cs index 2a7000aac6..20cd045bf5 100644 --- a/Rdmp.Core/Curation/Data/ConcreteFilter.cs +++ b/Rdmp.Core/Curation/Data/ConcreteFilter.cs @@ -169,9 +169,9 @@ public virtual void Check(ICheckNotifier notifier) } /// - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context,out string reason) { reason = null; - return FilterContainer?.ShouldBeReadOnly(out reason) ?? false; + return FilterContainer?.ShouldBeReadOnly(context,out reason) ?? false; } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/IMightBeReadOnly.cs b/Rdmp.Core/Curation/Data/IMightBeReadOnly.cs index efc344e9dd..13716b7948 100644 --- a/Rdmp.Core/Curation/Data/IMightBeReadOnly.cs +++ b/Rdmp.Core/Curation/Data/IMightBeReadOnly.cs @@ -17,5 +17,5 @@ public interface IMightBeReadOnly /// Returns true if changes to the container should be forbidden e.g. because the parent object is frozen (like ) /// /// - bool ShouldBeReadOnly(out string reason); + bool ShouldBeReadOnly(string context,out string reason); } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilterContainer.cs b/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilterContainer.cs index 78baa0b4ea..61b07b763e 100644 --- a/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilterContainer.cs +++ b/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilterContainer.cs @@ -54,7 +54,7 @@ public SpontaneouslyInventedFilterContainer(MemoryCatalogueRepository repo, ICon public override Catalogue GetCatalogueIfAny() => null; - public override bool ShouldBeReadOnly(out string reason) + public override bool ShouldBeReadOnly(string context, out string reason) { reason = null; return false; diff --git a/Rdmp.Core/Curation/FilterImporting/ParameterCollectionUIOptions.cs b/Rdmp.Core/Curation/FilterImporting/ParameterCollectionUIOptions.cs index 2bb0a77da0..b351ab9295 100644 --- a/Rdmp.Core/Curation/FilterImporting/ParameterCollectionUIOptions.cs +++ b/Rdmp.Core/Curation/FilterImporting/ParameterCollectionUIOptions.cs @@ -50,7 +50,7 @@ public ParameterCollectionUIOptions(string useCase, ICollectSqlParameters collec _createNewParameterDelegate = CreateNewParameterDefaultImplementation; if (collector is IMightBeReadOnly ro) - ReadOnly = ro.ShouldBeReadOnly(out _); + ReadOnly = ro.ShouldBeReadOnly(this.GetType().Name, out _); } diff --git a/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs b/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs index ce7aa73506..eae90ae836 100644 --- a/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs +++ b/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs @@ -342,7 +342,7 @@ internal ExtractionConfiguration(IDataExportRepository repository, DbDataReader /// public override string ToString() => Name; - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context,out string reason) { if (IsReleased) { diff --git a/Rdmp.Core/DataExport/Data/FilterContainer.cs b/Rdmp.Core/DataExport/Data/FilterContainer.cs index f5b805a4ce..2e425d3579 100644 --- a/Rdmp.Core/DataExport/Data/FilterContainer.cs +++ b/Rdmp.Core/DataExport/Data/FilterContainer.cs @@ -51,7 +51,7 @@ public override Catalogue GetCatalogueIfAny() return (Catalogue)sel?.ExtractableDataSet.Catalogue; } - public override bool ShouldBeReadOnly(out string reason) + public override bool ShouldBeReadOnly(string context, out string reason) { var ec = GetSelectedDataSetsRecursively()?.ExtractionConfiguration; @@ -61,7 +61,7 @@ public override bool ShouldBeReadOnly(out string reason) return false; } - return ec.ShouldBeReadOnly(out reason); + return ec.ShouldBeReadOnly(context, out reason); } /// diff --git a/Rdmp.Core/DataExport/Data/ProjectCohortIdentificationConfigurationAssociation.cs b/Rdmp.Core/DataExport/Data/ProjectCohortIdentificationConfigurationAssociation.cs index 4a462ff0ed..fe4ad090d5 100644 --- a/Rdmp.Core/DataExport/Data/ProjectCohortIdentificationConfigurationAssociation.cs +++ b/Rdmp.Core/DataExport/Data/ProjectCohortIdentificationConfigurationAssociation.cs @@ -122,10 +122,10 @@ public override string ToString() return assoc == null ? "Orphan Association" : assoc.Name; } - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context, out string reason) { reason = null; - return CohortIdentificationConfiguration?.ShouldBeReadOnly(out reason) ?? false; + return CohortIdentificationConfiguration?.ShouldBeReadOnly(context,out reason) ?? false; } /// diff --git a/Rdmp.Core/DataExport/Data/SelectedDataSets.cs b/Rdmp.Core/DataExport/Data/SelectedDataSets.cs index 5c8ed8ab66..402b137168 100644 --- a/Rdmp.Core/DataExport/Data/SelectedDataSets.cs +++ b/Rdmp.Core/DataExport/Data/SelectedDataSets.cs @@ -126,7 +126,7 @@ public SelectedDataSets(IDataExportRepository repository, ExtractionConfiguratio /// public override string ToString() => ExtractableDataSet.ToString(); - public bool ShouldBeReadOnly(out string reason) => ExtractionConfiguration.ShouldBeReadOnly(out reason); + public bool ShouldBeReadOnly(string context, out string reason) => ExtractionConfiguration.ShouldBeReadOnly(context, out reason); /// public string GetDeleteVerb() => "Remove"; diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql index 4610d3e03b..2a50d8948b 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql @@ -1,4 +1,4 @@ ---Version: 8.4.3 +--Version: 9.0.4 --Description: Add new metadata fields for catalogues if not exists (select 1 from sys.columns where name = 'IsTemplate' and OBJECT_NAME(object_id) = 'CohortIdentificationConfiguration') BEGIN diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReOrderAggregate.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReOrderAggregate.cs index ae5049b68a..5d31390190 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReOrderAggregate.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReOrderAggregate.cs @@ -64,7 +64,7 @@ private ExecuteCommandReOrderAggregate(IActivateItems activator, IOrderable targ _targetOrder = target; _insertOption = insertOption; - if (target is IMightBeReadOnly ro && ro.ShouldBeReadOnly(out var reason)) + if (target is IMightBeReadOnly ro && ro.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReOrderAggregateContainer.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReOrderAggregateContainer.cs index 915514c858..727e826010 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReOrderAggregateContainer.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReOrderAggregateContainer.cs @@ -35,7 +35,7 @@ public ExecuteCommandReOrderAggregateContainer(IActivateItems activator, if (_insertOption == InsertOption.Default) SetImpossible("Insert must be above/below"); - if (_targetParent.ShouldBeReadOnly(out var reason)) + if (_targetParent.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.UI/TestsAndSetup/ServicePropogation/RDMPSingleDatabaseObjectControl.cs b/Rdmp.UI/TestsAndSetup/ServicePropogation/RDMPSingleDatabaseObjectControl.cs index 9286ff82da..3e45fdc7be 100644 --- a/Rdmp.UI/TestsAndSetup/ServicePropogation/RDMPSingleDatabaseObjectControl.cs +++ b/Rdmp.UI/TestsAndSetup/ServicePropogation/RDMPSingleDatabaseObjectControl.cs @@ -118,7 +118,7 @@ public virtual void SetDatabaseObject(IActivateItems activator, T databaseObject if (databaseObject is IMightBeReadOnly ro) { - if (ro.ShouldBeReadOnly(out var reason)) + if (ro.ShouldBeReadOnly(this.GetType().Name, out var reason)) { _readonlyIndicator.Text = reason; Controls.Add(_readonlyIndicator); From 17459c87f02d1f6b5c6c73ccd06766b7710b389d Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 09:49:21 +0000 Subject: [PATCH 119/142] improve template namer --- .../RDMPMainForm.cs | 9 ++------- ...eCohortIdentificationConfigurationTemplate.cs | 16 +++++++++------- ...eTemplateCohortIdentificationConfiguration.cs | 15 ++++++++++++++- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/Application/ResearchDataManagementPlatform/RDMPMainForm.cs b/Application/ResearchDataManagementPlatform/RDMPMainForm.cs index b37a62d7e2..a018340aab 100644 --- a/Application/ResearchDataManagementPlatform/RDMPMainForm.cs +++ b/Application/ResearchDataManagementPlatform/RDMPMainForm.cs @@ -104,13 +104,8 @@ private void RDMPMainForm_Load(object sender, EventArgs e) _globalErrorCheckNotifier = exceptionCounter; _rdmpTopMenuStrip1.InjectButton(exceptionCounter); - try - { - _windowManager = new WindowManager(_theme, this, _refreshBus, dockPanel1, RepositoryLocator, exceptionCounter); - }catch(Exception ex) - { - Console.Write(ex.ToString()); - } + _windowManager = new WindowManager(_theme, this, _refreshBus, dockPanel1, RepositoryLocator, exceptionCounter); + SetItemActivator(_windowManager.ActivateItems); _rdmpTopMenuStrip1.SetWindowManager(_windowManager); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs index 4eef191774..b028519806 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs @@ -1,14 +1,11 @@ -using NPOI.OpenXmlFormats.Encryption; -using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.DataExport.Data; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.ReusableLibraryCode.Checks; using System; -using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System.Reflection.Metadata.Ecma335; namespace Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands { @@ -22,16 +19,21 @@ public ExecuteCommandCreateCohortIdentificationConfigurationTemplate(IBasicActiv _cic = cic; } + private string GenerateTemplateName(string name) + { + if (name.EndsWith(" Template")) return name; + return name + " Template"; + } + public override void Execute() { - //todo think about asking if we want to associate to he project if it already is var associations = _activator.RepositoryLocator.DataExportRepository.GetAllObjects(); var projectAssociations = associations.Where(a => a.CohortIdentificationConfiguration_ID == _cic.ID).ToList(); base.Execute(); var clone = _cic.CreateClone(ThrowImmediatelyCheckNotifier.Quiet); clone.IsTemplate = true; clone.Freeze(); - clone.Name = clone.Name.Replace("(Clone)", "") + " Template"; + clone.Name = GenerateTemplateName(clone.Name); clone.SaveToDatabase(); IMapsDirectlyToDatabaseTable selectedProject = null; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs index 94f6f13284..a2820e0c5e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs @@ -22,6 +22,19 @@ public ExecuteCommandUseTemplateCohortIdentificationConfiguration(IBasicActivate _cic = cic; } + private string RenameTemplateForUse(string name) + { + if (name.EndsWith("Template")) + { + name = name.Substring(0,name.Length - 8); + } + while(name.EndsWith(" ")) + { + name = name.Substring(0,name.Length - 1); + } + return name; + } + public override void Execute() { var associations = _activator.RepositoryLocator.DataExportRepository.GetAllObjects(); @@ -58,7 +71,7 @@ public override void Execute() base.Execute(); var clone = _cic.CreateClone(ThrowImmediatelyCheckNotifier.Quiet); clone.IsTemplate = false; - clone.Name = clone.Name.Replace(" Template", ""); + clone.Name = RenameTemplateForUse(clone.Name); clone.SaveToDatabase(); if (selectedProject != null) { From af34f3964407def5ba3a8d83c1c6f28322c64561 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 10:39:00 +0000 Subject: [PATCH 120/142] add tests --- ...dentificationConfigurationTemplateTests.cs | 97 +++++++++++++++++++ .../TestCommandsAreSupported.cs | 2 + .../Curation/Unit/IMightBeReadOnlyTests.cs | 42 +++++--- ...hortIdentificationConfigurationTemplate.cs | 30 ++++-- ...mplateCohortIdentificationConfiguration.cs | 20 ++-- 5 files changed, 160 insertions(+), 31 deletions(-) create mode 100644 Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs diff --git a/Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs b/Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs new file mode 100644 index 0000000000..d74e3a0762 --- /dev/null +++ b/Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs @@ -0,0 +1,97 @@ +using MathNet.Numerics; +using NUnit.Framework; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tests.Common; + +namespace Rdmp.Core.Tests.CohortCreation +{ + public class CohortIdentificationConfigurationTemplateTests: DatabaseTests + { + private static Random random = new Random(); + private static string RandomString(int length) + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + return new string(Enumerable.Repeat(chars, length) + .Select(s => s[random.Next(s.Length)]).ToArray()); + } + + private CohortIdentificationConfiguration CreateBaseCIC(string name) + { + var cic = new CohortIdentificationConfiguration(CatalogueRepository, name); + cic.CreateRootContainerIfNotExists(); + cic.SaveToDatabase(); + return cic; + } + [Test] + public void CreateTemplateFromCIC_Test() + { + var cohortName = RandomString(10); + var cic = CreateBaseCIC(cohortName); + var cmd = new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(new ThrowImmediatelyActivator(RepositoryLocator), cic); + Assert.DoesNotThrow(()=>cmd.Execute()); + var foundTemplate = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName} Template"); + Assert.That(foundTemplate.Length,Is.EqualTo(1)); + } + + [Test] + public void CreateTemplateFromCICWithProject_Test() + { + var cohortName = RandomString(10); + var cic = CreateBaseCIC(cohortName); + var project = new Project(DataExportRepository, RandomString(10)); + project.SaveToDatabase(); + var cmd = new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(new ThrowImmediatelyActivator(RepositoryLocator), cic); + cmd.SetTarget(project); + Assert.DoesNotThrow(() => cmd.Execute()); + var foundTemplate = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName} Template"); + Assert.That(foundTemplate.Length, Is.EqualTo(1)); + var associations = RepositoryLocator.DataExportRepository.GetAllObjects().Where(a => a.CohortIdentificationConfiguration_ID == foundTemplate[0].ID).ToList(); + Assert.That(associations.Count, Is.EqualTo(1)); + Assert.That(associations[0].Project.Name, Is.EqualTo(project.Name)); + } + + [Test] + public void CreateCICFromTemplate_Test() + { + var cohortName = RandomString(10); + var cic = CreateBaseCIC(cohortName); + var ciccmd = new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(new ThrowImmediatelyActivator(RepositoryLocator), cic); + Assert.DoesNotThrow(() => ciccmd.Execute()); + var template = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName} Template").First(); + + var cmd = new ExecuteCommandUseTemplateCohortIdentificationConfiguration(new ThrowImmediatelyActivator(RepositoryLocator), template); + Assert.DoesNotThrow(() => cmd.Execute()); + var cics = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName}"); + Assert.That(cics.Length, Is.EqualTo(2)); + } + + [Test] + public void CreateCICFromTemplateWithProject_Test() + { + var cohortName = RandomString(10); + var cic = CreateBaseCIC(cohortName); + var ciccmd = new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(new ThrowImmediatelyActivator(RepositoryLocator), cic); + Assert.DoesNotThrow(() => ciccmd.Execute()); + var template = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName} Template").First(); + var project = new Project(DataExportRepository, RandomString(10)); + project.SaveToDatabase(); + var cmd = new ExecuteCommandUseTemplateCohortIdentificationConfiguration(new ThrowImmediatelyActivator(RepositoryLocator), template); + cmd.SetTarget(project); + Assert.DoesNotThrow(() => cmd.Execute()); + var cics = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName}"); + Assert.That(cics.Length, Is.EqualTo(2)); + var associations = RepositoryLocator.DataExportRepository.GetAllObjects().Where(a => cics.Select(c => c.ID).Contains(a.CohortIdentificationConfiguration_ID)).ToList(); + Assert.That(associations.Count, Is.EqualTo(1)); + Assert.That(associations[0].Project.Name, Is.EqualTo(project.Name)); + } + } +} diff --git a/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs b/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs index 67ae4e50c1..5f9a5cd1c8 100644 --- a/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs +++ b/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs @@ -38,6 +38,8 @@ public void Init() [TestCase(typeof(ExecuteCommandCreateNewCatalogueByImportingFile))] [TestCase(typeof(ExecuteCommandCreateNewCatalogueFromTableInfo))] [TestCase(typeof(ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration))] + [TestCase(typeof(ExecuteCommandUseTemplateCohortIdentificationConfiguration))] + [TestCase(typeof(ExecuteCommandCreateCohortIdentificationConfigurationTemplate))] [TestCase(typeof(ExecuteCommandCreateNewCohortFromCatalogue))] [TestCase(typeof(ExecuteCommandCreateNewCohortFromFile))] [TestCase(typeof(ExecuteCommandCreateNewCohortFromTable))] diff --git a/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs b/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs index 75f34dfb71..4b3d322895 100644 --- a/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs +++ b/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs @@ -22,7 +22,7 @@ public void IsReadonly_AggregateFilterContainer() { //im probably an orphan var c = WhenIHaveA(); - Assert.That(c.ShouldBeReadOnly(out _), Is.False); + Assert.That(c.ShouldBeReadOnly(null,out _), Is.False); //now I am in a cic var cic = WhenIHaveA(); @@ -30,12 +30,12 @@ public void IsReadonly_AggregateFilterContainer() cic.CreateRootContainerIfNotExists(); cic.RootCohortAggregateContainer.AddChild(c.GetAggregate(), 0); - Assert.That(c.ShouldBeReadOnly(out _), Is.False); + Assert.That(c.ShouldBeReadOnly(null, out _), Is.False); cic.Frozen = true; Assert.Multiple(() => { - Assert.That(c.ShouldBeReadOnly(out var reason)); + Assert.That(c.ShouldBeReadOnly(null, out var reason)); Assert.That(reason, Is.EqualTo("fff is Frozen")); }); @@ -45,17 +45,17 @@ public void IsReadonly_AggregateFilterContainer() public void IsReadonly_ExtractionFilterContainer() { var c = WhenIHaveA(); - Assert.That(c.ShouldBeReadOnly(out _), Is.False); + Assert.That(c.ShouldBeReadOnly(null, out _), Is.False); var ec = c.GetSelectedDataSetIfAny().ExtractionConfiguration; - Assert.That(c.ShouldBeReadOnly(out _), Is.False); + Assert.That(c.ShouldBeReadOnly(null, out _), Is.False); ec.Name = "lll"; ec.IsReleased = true; Assert.Multiple(() => { - Assert.That(c.ShouldBeReadOnly(out var reason)); + Assert.That(c.ShouldBeReadOnly(null, out var reason)); Assert.That(reason, Is.EqualTo("lll has already been released")); }); @@ -66,17 +66,35 @@ public void IsReadonly_SpontaneousContainer() { var memoryrepo = new MemoryCatalogueRepository(); var c = new SpontaneouslyInventedFilterContainer(memoryrepo, null, null, FilterContainerOperation.AND); - Assert.That(c.ShouldBeReadOnly(out _), Is.False, + Assert.That(c.ShouldBeReadOnly(null, out _), Is.False, "Spont containers should never be in UI but let's not tell the programmer they shouldn't be edited"); } + [Test] + public void IsReadOnly_TemplateCohortIdentificationConfiguration() + { + var cic = WhenIHaveA(); + cic.IsTemplate = true; + Assert.That(cic.ShouldBeReadOnly(null, out _), Is.True); + + } + + [Test] + public void IsReadOnly_TemplateCohortIdentificationConfiguration_Rename() + { + var cic = WhenIHaveA(); + cic.IsTemplate = true; + Assert.That(cic.ShouldBeReadOnly("ExecuteCommandRename", out _), Is.False); + + } + [Test] public void IsReadonly_AggregateFilter() { //im probably an orphan var f = WhenIHaveA(); - Assert.That(f.ShouldBeReadOnly(out _), Is.False); + Assert.That(f.ShouldBeReadOnly(null, out _), Is.False); //now I am in a cic var cic = WhenIHaveA(); @@ -84,12 +102,12 @@ public void IsReadonly_AggregateFilter() cic.CreateRootContainerIfNotExists(); cic.RootCohortAggregateContainer.AddChild(f.GetAggregate(), 0); - Assert.That(f.ShouldBeReadOnly(out _), Is.False); + Assert.That(f.ShouldBeReadOnly(null, out _), Is.False); cic.Frozen = true; Assert.Multiple(() => { - Assert.That(f.ShouldBeReadOnly(out var reason)); + Assert.That(f.ShouldBeReadOnly(null, out var reason)); Assert.That(reason, Is.EqualTo("fff is Frozen")); }); @@ -99,14 +117,14 @@ public void IsReadonly_AggregateFilter() public void IsReadonly_DeployedExtractionFilter() { var f = WhenIHaveA(); - Assert.That(f.ShouldBeReadOnly(out _), Is.False); + Assert.That(f.ShouldBeReadOnly(null, out _), Is.False); var ec = ((FilterContainer)f.FilterContainer).GetSelectedDataSetIfAny().ExtractionConfiguration; ec.Name = "lll"; ec.IsReleased = true; Assert.Multiple(() => { - Assert.That(f.ShouldBeReadOnly(out var reason)); + Assert.That(f.ShouldBeReadOnly(null, out var reason)); Assert.That(reason, Is.EqualTo("lll has already been released")); }); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs index b028519806..db30568aa5 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs @@ -9,11 +9,12 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands { - public class ExecuteCommandCreateCohortIdentificationConfigurationTemplate: BasicCommandExecution,IAtomicCommandWithTarget + public class ExecuteCommandCreateCohortIdentificationConfigurationTemplate : BasicCommandExecution, IAtomicCommandWithTarget { private CohortIdentificationConfiguration _cic; + private IMapsDirectlyToDatabaseTable _selectedProject; private IBasicActivateItems _activator; - public ExecuteCommandCreateCohortIdentificationConfigurationTemplate(IBasicActivateItems activator, CohortIdentificationConfiguration cic): base(activator) + public ExecuteCommandCreateCohortIdentificationConfigurationTemplate(IBasicActivateItems activator, CohortIdentificationConfiguration cic) : base(activator) { _activator = activator; _cic = cic; @@ -21,6 +22,7 @@ public ExecuteCommandCreateCohortIdentificationConfigurationTemplate(IBasicActiv private string GenerateTemplateName(string name) { + if (name.EndsWith(" (Clone)")) name = name.Substring(0, name.Length - 8); if (name.EndsWith(" Template")) return name; return name + " Template"; } @@ -35,15 +37,15 @@ public override void Execute() clone.Freeze(); clone.Name = GenerateTemplateName(clone.Name); clone.SaveToDatabase(); - IMapsDirectlyToDatabaseTable selectedProject = null; - if (_activator.IsInteractive && projectAssociations.Any()) { + if (_activator.IsInteractive && projectAssociations.Any()) + { if (projectAssociations.Count > 1) { //multiple, make them pick if (_activator.YesNo("This Cohort Configuration is associated with multiple Projects. Would you like to associate this Template with one of them?", "Associate with a Project")) { - selectedProject = _activator.SelectOne("Select a Project to associate this Template with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + _selectedProject = _activator.SelectOne("Select a Project to associate this Template with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); } } else if (projectAssociations.Count == 1) @@ -52,7 +54,7 @@ public override void Execute() //ask them if they want to use this one if (_activator.YesNo($"This cohort configuration is already associated with the {projectAssociations.First().Project.Name} project. Would you like to associate this Template with this project?", "Use Existing Project")) { - selectedProject = projectAssociations.First().Project; + _selectedProject = projectAssociations.First().Project; } } else @@ -60,21 +62,29 @@ public override void Execute() //ask if they want to use it in a project if (_activator.YesNo("Would you like to associate this Template with a project?", "Associate with a Project")) { - selectedProject = _activator.SelectOne("Select a Project to associate this Template with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + _selectedProject = _activator.SelectOne("Select a Project to associate this Template with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); } } } + if (_selectedProject != null) + { + var cmd = new ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(_activator); + cmd.SetTarget(clone); + cmd.SetTarget((Project)_selectedProject); + cmd.Execute(); + } Publish(clone); Emphasise(clone); } public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) { - if(target is not CohortIdentificationConfiguration) + if (target is not CohortIdentificationConfiguration && target is not Project) { - throw new Exception("Provided database entity was not a CohortIdentificationConfiguration."); + throw new Exception("Provided database entity was not a CohortIdentificationConfiguration or a Project."); } - _cic = target as CohortIdentificationConfiguration; + if (target is Project p) _selectedProject = p; + else if (target is CohortIdentificationConfiguration c) _cic = c; return this; } } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs index a2820e0c5e..c9eac51120 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs @@ -15,6 +15,7 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands public class ExecuteCommandUseTemplateCohortIdentificationConfiguration : BasicCommandExecution, IAtomicCommandWithTarget { private CohortIdentificationConfiguration _cic; + private IMapsDirectlyToDatabaseTable _selectedProject; private IBasicActivateItems _activator; public ExecuteCommandUseTemplateCohortIdentificationConfiguration(IBasicActivateItems activator, CohortIdentificationConfiguration cic) : base(activator) { @@ -24,6 +25,7 @@ public ExecuteCommandUseTemplateCohortIdentificationConfiguration(IBasicActivate private string RenameTemplateForUse(string name) { + if (name.EndsWith(" (Clone)")) name = name.Substring(0, name.Length - 8); if (name.EndsWith("Template")) { name = name.Substring(0,name.Length - 8); @@ -39,7 +41,6 @@ public override void Execute() { var associations = _activator.RepositoryLocator.DataExportRepository.GetAllObjects(); var projectAssociations = associations.Where(a => a.CohortIdentificationConfiguration_ID == _cic.ID).ToList(); - IMapsDirectlyToDatabaseTable selectedProject = null; if (_activator.IsInteractive && projectAssociations.Any()) { if (projectAssociations.Count > 1) @@ -47,7 +48,7 @@ public override void Execute() //multiple, make them pick if (_activator.YesNo("This Template is associated with multiple Projects. Would you like to associate this cohort configuration with one of them?", "Associate with a Project")) { - selectedProject = _activator.SelectOne("Select a Project to associate this cohort with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + _selectedProject = _activator.SelectOne("Select a Project to associate this cohort with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); } } else if (projectAssociations.Count == 1) @@ -56,7 +57,7 @@ public override void Execute() //ask them if they want to use this one if (_activator.YesNo($"This template is already associated with the {projectAssociations.First().Project.Name} project. Would you like to associate this cohort configuration with this project?", "Use Existing Project")) { - selectedProject = projectAssociations.First().Project; + _selectedProject = projectAssociations.First().Project; } } else @@ -64,7 +65,7 @@ public override void Execute() //ask if they want to use it in a project if (_activator.YesNo("Would you like to associate this cohort configuration with a project?", "Associate with a Project")) { - selectedProject = _activator.SelectOne("Select a Project to associate this cohort with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + _selectedProject = _activator.SelectOne("Select a Project to associate this cohort with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); } } } @@ -73,11 +74,11 @@ public override void Execute() clone.IsTemplate = false; clone.Name = RenameTemplateForUse(clone.Name); clone.SaveToDatabase(); - if (selectedProject != null) + if (_selectedProject != null) { var cmd = new ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(_activator); cmd.SetTarget(clone); - cmd.SetTarget((Project)selectedProject); + cmd.SetTarget((Project)_selectedProject); cmd.Execute(); } Publish(clone); @@ -86,11 +87,12 @@ public override void Execute() public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) { - if (target is not CohortIdentificationConfiguration cic || !cic.IsTemplate) + if ((target is not CohortIdentificationConfiguration && target is not Project) || (target is CohortIdentificationConfiguration cic && !cic.IsTemplate)) { - throw new Exception("Provided database entity was not a CohortIdentificationConfiguration."); + throw new Exception("Provided database entity was not a CohortIdentificationConfiguration or a Project."); } - _cic = target as CohortIdentificationConfiguration; + if (target is Project p) _selectedProject = p; + else if (target is CohortIdentificationConfiguration c) _cic = c; return this; } } From e41695190c653acca7e3a63369cbe97f08a325b9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 10:40:51 +0000 Subject: [PATCH 121/142] update version number --- CHANGELOG.md | 2 +- SharedAssemblyInfo.cs | 6 +++--- rdmp-client.xml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 971a83ec2e..a3e5b40fe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [9.0.4] - Unreleased +## [9.1.0] - Unreleased - Fix bug with duplicate searchables - Add the ability to Template Cohort Identification Configurations (see [Documentation\cohorts\CohortIdentificationConfigurationTemplates.md]) diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 401bbb05b0..c271079044 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("9.0.4")] -[assembly: AssemblyFileVersion("9.0.4")] -[assembly: AssemblyInformationalVersion("9.0.4")] +[assembly: AssemblyVersion("9.1.0")] +[assembly: AssemblyFileVersion("9.1.0")] +[assembly: AssemblyInformationalVersion("9.1.0")] diff --git a/rdmp-client.xml b/rdmp-client.xml index 06b5287257..f81a73d09e 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -1,7 +1,7 @@ - 9.0.3.0 - https://github.com/HicServices/RDMP/releases/download/v9.0.3/rdmp-9.0.3-client.zip + 9.1.0.0 + https://github.com/HicServices/RDMP/releases/download/v9.1.0/rdmp-9.1.0-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true From 2aa7efd4da762054d6407361ade15c68164717d8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 11:00:51 +0000 Subject: [PATCH 122/142] codeql updates --- .../CohortIdentificationConfigurationTemplateTests.cs | 2 +- Rdmp.Core/CommandExecution/AtomicCommandFactory.cs | 2 +- ...dCreateCohortIdentificationConfigurationTemplate.cs | 2 +- .../Data/Cohort/CohortIdentificationConfiguration.cs | 10 ++++------ 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs b/Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs index d74e3a0762..efee93e1b8 100644 --- a/Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs +++ b/Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs @@ -16,7 +16,7 @@ namespace Rdmp.Core.Tests.CohortCreation { public class CohortIdentificationConfigurationTemplateTests: DatabaseTests { - private static Random random = new Random(); + private static readonly Random random = new(); private static string RandomString(int length) { const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index d621b3548d..a6d39d3d22 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -470,7 +470,7 @@ public IEnumerable CreateCommands(object o) yield return new ExecuteCommandViewData(_activator, cic, ViewType.All, null, true) { Weight = -99.7f }; yield return new ExecuteCommandViewData(_activator, cic, ViewType.All, null, false) { Weight = -99.6f }; - if (cic.IsTemplate) + if (cic != null && cic.IsTemplate) { yield return new ExecuteCommandUseTemplateCohortIdentificationConfiguration(_activator, cic) { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs index db30568aa5..966e21b400 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs @@ -13,7 +13,7 @@ public class ExecuteCommandCreateCohortIdentificationConfigurationTemplate : Bas { private CohortIdentificationConfiguration _cic; private IMapsDirectlyToDatabaseTable _selectedProject; - private IBasicActivateItems _activator; + private readonly IBasicActivateItems _activator; public ExecuteCommandCreateCohortIdentificationConfigurationTemplate(IBasicActivateItems activator, CohortIdentificationConfiguration cic) : base(activator) { _activator = activator; diff --git a/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs b/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs index 6982d425f9..86596eb5a5 100644 --- a/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs @@ -285,14 +285,12 @@ public void CreateRootContainerIfNotExists() public bool ShouldBeReadOnly(string context, out string reason) { - if (IsTemplate) + if (IsTemplate && context == "ExecuteCommandRename") { - if (context == "ExecuteCommandRename") - { - reason = null; - return false; - } + reason = null; + return false; } + if (Frozen) { reason = $"{Name} is Frozen"; From a1dd0ee1ff0c76595e247bcc225f1ee5e31586f6 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 11:23:28 +0000 Subject: [PATCH 123/142] fix test --- Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs b/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs index 4b3d322895..5bd22aeeff 100644 --- a/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs +++ b/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs @@ -75,6 +75,7 @@ public void IsReadOnly_TemplateCohortIdentificationConfiguration() { var cic = WhenIHaveA(); cic.IsTemplate = true; + cic.Frozen = true; Assert.That(cic.ShouldBeReadOnly(null, out _), Is.True); } From 0a1decdc556644b1016f9519641ae7a3fca886ed Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 14:44:56 +0000 Subject: [PATCH 124/142] add missing documentation --- ...ortIdentificationConfigurationTemplates.md | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md diff --git a/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md b/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md new file mode 100644 index 0000000000..7780418db0 --- /dev/null +++ b/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md @@ -0,0 +1,35 @@ +## Cohort Identification Configuration Templates + +Cohort Identification Configurations (CICs) can be turned into Templates for future re-use. +Templates offer certain functionality that differs from standard CICs. They: +* Cannot be edited +* Cannot be used to commit a cohort +* Are stored in a separate Templates section of the Cohort Builder + +Templates are designed to be a base to build your CIC from, saving you from having to copy, add, and edit boilerplate CIC configuration each and every time. + +### Creating a Template + +To create a new CIC Template: +1. Begin by creating a new CIC (or use an existing CIC). +2.Update your CIC to create the configuration you wish to template. +3. Once ready to turn this CIC into a template, right click on the CIC in the cohort builder tree. +4. Select the option "Create Cohort Identification Template". +5. If your CIC is associated with a project, it will ask you some questions about how you would like to associate the template +6. Congratulations, You've just made a CIC template + + +### Using a Template + +To use a CIC Template: +1. Right click on an existing CIC Template +2. Select the "Use Template Cohort Identification Configuration" option +3. If your template CIC is associated with a project, it will ask you some questions about how you would like to associate the new CIC +4. Congratulations, You've just made a new CIC from a template + + +### Associating with a Project + +Just like associating a CIC with a Project, CIC Templates can also be associated with Projects. +While these templates can still be used outside the associated project, it does allow you to see the template from within the Project view. +This may be helpful to keep track of what templates are typically used with each project \ No newline at end of file From 3de5ae1b07b1f47e8a2a25eb4d91fef753907e62 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 14:47:04 +0000 Subject: [PATCH 125/142] tidy up from review --- ...teCommandUseTemplateCohortIdentificationConfiguration.cs | 2 +- .../CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql | 2 +- Rdmp.Core/Providers/CatalogueChildProvider.cs | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs index c9eac51120..643bc692fa 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs @@ -16,7 +16,7 @@ public class ExecuteCommandUseTemplateCohortIdentificationConfiguration : BasicC { private CohortIdentificationConfiguration _cic; private IMapsDirectlyToDatabaseTable _selectedProject; - private IBasicActivateItems _activator; + private readonly IBasicActivateItems _activator; public ExecuteCommandUseTemplateCohortIdentificationConfiguration(IBasicActivateItems activator, CohortIdentificationConfiguration cic) : base(activator) { _activator = activator; diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql index 2a50d8948b..4bccc079cf 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql @@ -1,4 +1,4 @@ ---Version: 9.0.4 +--Version: 9.1.0 --Description: Add new metadata fields for catalogues if not exists (select 1 from sys.columns where name = 'IsTemplate' and OBJECT_NAME(object_id) = 'CohortIdentificationConfiguration') BEGIN diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 097aac61c1..83faac340a 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -147,8 +147,6 @@ public class CatalogueChildProvider : ICoreChildProvider public FolderNode DatasetRootFolder { get; set; } public FolderNode CohortIdentificationConfigurationRootFolder { get; set; } - //public FolderNode TemplateCohortIdentificationConfigurationRootFolder { get; set; } - public AllTemplateCohortIdentificationConfigurationsNode AllTemplateCohortIdentificationConfigurationsNode { get; set; } public FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; set; } @@ -423,10 +421,6 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] .Where(p => p.ReferencedObjectType.Equals(nameof(AggregateConfiguration))) .Select(r => r.ReferencedObjectID)); - //TemplateCohortIdentificationConfigurationRootFolder = FolderHelper.BuildFolderTree(AllTemplateCohortIdentificationConfigurations); - //AddChildren(TemplateCohortIdentificationConfigurationRootFolder, - // new DescendancyList(TemplateCohortIdentificationConfigurationRootFolder)); - AllTemplateCohortIdentificationConfigurationsNode = new AllTemplateCohortIdentificationConfigurationsNode(); var templateCICTree = FolderHelper.BuildFolderTree(AllTemplateCohortIdentificationConfigurations); AddChildren(templateCICTree, new DescendancyList(AllTemplateCohortIdentificationConfigurationsNode)); From 79b15a9b5cf2d2edd12f633e21ec9d0487d0a2a1 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 15:10:53 +0000 Subject: [PATCH 126/142] update docs --- .../Cohorts/CohortIdentificationConfigurationTemplates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md b/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md index 7780418db0..8d2ad60430 100644 --- a/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md +++ b/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md @@ -30,6 +30,6 @@ To use a CIC Template: ### Associating with a Project -Just like associating a CIC with a Project, CIC Templates can also be associated with Projects. +Just like associating a CIC with a [Project]:../CodeTutorials/Glossary.md#Project, CIC Templates can also be associated with Projects. While these templates can still be used outside the associated project, it does allow you to see the template from within the Project view. This may be helpful to keep track of what templates are typically used with each project \ No newline at end of file From 8bad511bec3560b0058897c460047df6ac66cfa5 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 15:28:23 +0000 Subject: [PATCH 127/142] update docs --- .../Cohorts/CohortIdentificationConfigurationTemplates.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md b/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md index 8d2ad60430..c62f363736 100644 --- a/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md +++ b/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md @@ -24,12 +24,12 @@ To create a new CIC Template: To use a CIC Template: 1. Right click on an existing CIC Template 2. Select the "Use Template Cohort Identification Configuration" option -3. If your template CIC is associated with a project, it will ask you some questions about how you would like to associate the new CIC +3. If your template CIC is associated with a [Project]:../CodeTutorials/Glossary.md#Project, it will ask you some questions about how you would like to associate the new CIC 4. Congratulations, You've just made a new CIC from a template -### Associating with a Project +### Associating with a [Project]:../CodeTutorials/Glossary.md#Project Just like associating a CIC with a [Project]:../CodeTutorials/Glossary.md#Project, CIC Templates can also be associated with Projects. -While these templates can still be used outside the associated project, it does allow you to see the template from within the Project view. -This may be helpful to keep track of what templates are typically used with each project \ No newline at end of file +While these templates can still be used outside the associated [Project]:../CodeTutorials/Glossary.md#Project, it does allow you to see the template from within the [Project]:../CodeTutorials/Glossary.md#Project view. +This may be helpful to keep track of what templates are typically used with each [Project]:../CodeTutorials/Glossary.md#Project \ No newline at end of file From d216ae723fbd65bb372f8d53260baaffdb3ce8fb Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 11 Nov 2025 15:33:29 +0000 Subject: [PATCH 128/142] add links --- .../CohortIdentificationConfigurationTemplates.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md b/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md index c62f363736..e5eea2410e 100644 --- a/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md +++ b/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md @@ -15,7 +15,7 @@ To create a new CIC Template: 2.Update your CIC to create the configuration you wish to template. 3. Once ready to turn this CIC into a template, right click on the CIC in the cohort builder tree. 4. Select the option "Create Cohort Identification Template". -5. If your CIC is associated with a project, it will ask you some questions about how you would like to associate the template +5. If your CIC is associated with a [Project], it will ask you some questions about how you would like to associate the template 6. Congratulations, You've just made a CIC template @@ -24,12 +24,15 @@ To create a new CIC Template: To use a CIC Template: 1. Right click on an existing CIC Template 2. Select the "Use Template Cohort Identification Configuration" option -3. If your template CIC is associated with a [Project]:../CodeTutorials/Glossary.md#Project, it will ask you some questions about how you would like to associate the new CIC +3. If your template CIC is associated with a [Project], it will ask you some questions about how you would like to associate the new CIC 4. Congratulations, You've just made a new CIC from a template -### Associating with a [Project]:../CodeTutorials/Glossary.md#Project +### Associating with a [Project] -Just like associating a CIC with a [Project]:../CodeTutorials/Glossary.md#Project, CIC Templates can also be associated with Projects. -While these templates can still be used outside the associated [Project]:../CodeTutorials/Glossary.md#Project, it does allow you to see the template from within the [Project]:../CodeTutorials/Glossary.md#Project view. -This may be helpful to keep track of what templates are typically used with each [Project]:../CodeTutorials/Glossary.md#Project \ No newline at end of file +Just like associating a CIC with a [Project], CIC Templates can also be associated with [Project]s. +While these templates can still be used outside the associated [Project], it does allow you to see the template from within the [Project] view. +This may be helpful to keep track of what templates are typically used with each [Project] + + +[Project]: ../CodeTutorials/Glossary.md#Project \ No newline at end of file From c6a06ba5b1f850ae314fdd66bf5e4d23c5b0478e Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 12 Nov 2025 14:08:37 +0000 Subject: [PATCH 129/142] update delete commit requirements --- CHANGELOG.md | 1 + .../AtomicCommands/ExecuteCommandDelete.cs | 15 +++------------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2052c0f96b..1a1c260c1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.0.4] - Unreleased - Fix bug with duplicate searchables +- Require all deletes to enter a commit message when using the commit system ## [9.0.3] - 2025-11-03 - Improve checking for default pipelines diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs index ef4a191c58..c6ed6fa570 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs @@ -80,18 +80,9 @@ private string GetDeleteVerbIfAny() public override void Execute() { base.Execute(); - - // if the thing we are deleting is important and sensitive then we should use a transaction - if (_deletables.Count > 1 || ShouldUseTransactionsWhenDeleting(_deletables.FirstOrDefault())) - { - ExecuteWithCommit(ExecuteImpl, GetDescription(), - _deletables.OfType().ToArray()); - PublishNearest(); - } - else - { - ExecuteImpl(); - } + ExecuteWithCommit(ExecuteImpl, GetDescription(), + _deletables.OfType().ToArray()); + PublishNearest(); } private static bool ShouldUseTransactionsWhenDeleting(IDeleteable deletable) => From 5079de646f01b59085ceb74bec5f5b86b594f1e3 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 14 Nov 2025 10:54:34 +0000 Subject: [PATCH 130/142] tidy up --- Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs | 2 +- Rdmp.UI/Collections/CatalogueCollectionUI.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs index 541c79a602..af1e8be113 100644 --- a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs +++ b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs @@ -29,7 +29,7 @@ public static class UserSettings public static bool ShowFlatLists { - get => AppSettings.GetValueOrDefault("ShowFlatLists", true); + get => AppSettings.GetValueOrDefault("ShowFlatLists", false); set => AppSettings.AddOrUpdateValue("ShowFlatLists", value); } public static bool UseLocalFileSystem diff --git a/Rdmp.UI/Collections/CatalogueCollectionUI.cs b/Rdmp.UI/Collections/CatalogueCollectionUI.cs index 18a6c30136..f1508e01a8 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionUI.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionUI.cs @@ -203,7 +203,7 @@ public void ApplyFilters() //reset to flat view tlvCatalogues.RemoveObject(Activator.CoreChildProvider.CatalogueRootFolder); tlvCatalogues.AddObjects(Activator.CoreChildProvider.AllCatalogues); - //tlvCatalogues.RefreshObjects(Activator.CoreChildProvider.AllCatalogues); + tlvCatalogues.RefreshObjects(Activator.CoreChildProvider.AllCatalogues); _renderingFlatFiew = true; } else if(!UserSettings.ShowFlatLists && _renderingFlatFiew) From bbd973813edc5a04491d5f727b0e3626d2ede9df Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 17 Nov 2025 08:44:27 +0000 Subject: [PATCH 131/142] prevent drag and drop --- Rdmp.UI/Collections/CatalogueCollectionUI.cs | 1 - ...ionWhenTargetIsCohortAggregateContainer.cs | 26 ++++++++++++--- .../RDMPCommandExecutionFactory.cs | 32 +++++++++++++++---- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/Rdmp.UI/Collections/CatalogueCollectionUI.cs b/Rdmp.UI/Collections/CatalogueCollectionUI.cs index f1508e01a8..78f53fa55c 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionUI.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionUI.cs @@ -203,7 +203,6 @@ public void ApplyFilters() //reset to flat view tlvCatalogues.RemoveObject(Activator.CoreChildProvider.CatalogueRootFolder); tlvCatalogues.AddObjects(Activator.CoreChildProvider.AllCatalogues); - tlvCatalogues.RefreshObjects(Activator.CoreChildProvider.AllCatalogues); _renderingFlatFiew = true; } else if(!UserSettings.ShowFlatLists && _renderingFlatFiew) diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCohortAggregateContainer.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCohortAggregateContainer.cs index e340e17dcd..6df0b4581f 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCohortAggregateContainer.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCohortAggregateContainer.cs @@ -8,8 +8,11 @@ using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.Providers; using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.ItemActivation; +using System.Linq; namespace Rdmp.UI.CommandExecution.Proposals; @@ -39,8 +42,23 @@ public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, { //source is catalogue case CatalogueCombineable sourceCatalogueCombineable: - return new ExecuteCommandAddCatalogueToCohortIdentificationSetContainer(ItemActivator, - sourceCatalogueCombineable, targetCohortAggregateContainer); + { + if (sourceCatalogueCombineable.Catalogue.IsProjectSpecific(ItemActivator.RepositoryLocator.DataExportRepository)) + { + var dx = (DataExportChildProvider)ItemActivator.CoreChildProvider; + var cic = targetCohortAggregateContainer.GetCohortIdentificationConfiguration(); + var cicProjAssociations = dx.AllProjectAssociatedCics.Where(c => c.CohortIdentificationConfiguration_ID == cic.ID).ToArray().Select(a => a.Project); + var extractableDatasets = ItemActivator.RepositoryLocator.DataExportRepository.GetAllObjectsWithParent(sourceCatalogueCombineable.Catalogue).ToList(); + var catalogueProjects = extractableDatasets.SelectMany(e => e.Projects); + if (!catalogueProjects.Any(c => cicProjAssociations.Contains(c))) + { + return null; + } + + } + return new ExecuteCommandAddCatalogueToCohortIdentificationSetContainer(ItemActivator, + sourceCatalogueCombineable, targetCohortAggregateContainer); + } //source is aggregate //if it is not already involved in cohort identification case AggregateConfigurationCombineable sourceAggregateCommand @@ -90,8 +108,8 @@ when sourceCohortAggregateContainerCommand.ParentContainerIfAny.Equals(targetCoh sourceCohortAggregateContainerCommand, targetCohortAggregateContainer); //it's being dragged above/below a container (reorder) case CohortAggregateContainerCombineable sourceCohortAggregateContainerCommand: - return new ExecuteCommandReOrderAggregateContainer(ItemActivator, sourceCohortAggregateContainerCommand, - targetCohortAggregateContainer, insertOption); + return new ExecuteCommandReOrderAggregateContainer(ItemActivator, sourceCohortAggregateContainerCommand, + targetCohortAggregateContainer, insertOption); } return null; diff --git a/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs b/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs index 56428ec692..b3df6ba4b7 100644 --- a/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs +++ b/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs @@ -4,14 +4,13 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Linq; using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.Providers; using Rdmp.Core.Providers.Nodes; using Rdmp.Core.Repositories; using Rdmp.Core.Repositories.Construction; @@ -19,6 +18,10 @@ using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.CommandExecution.Proposals; using Rdmp.UI.ItemActivation; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Windows.Forms; namespace Rdmp.UI.CommandExecution; @@ -162,10 +165,25 @@ private ICommandExecution CreateWhenTargetIsJoinableCollectionNode(ICombineToMak return new ExecuteCommandConvertAggregateConfigurationToPatientIndexTable(_activator, sourceAggregateConfigurationCombineable, targetJoinableCollectionNode.Configuration); - return cmd is CatalogueCombineable sourceCatalogueCombineable - ? new ExecuteCommandAddCatalogueToCohortIdentificationAsPatientIndexTable(_activator, - sourceCatalogueCombineable, targetJoinableCollectionNode.Configuration) - : (ICommandExecution)null; + if (cmd is CatalogueCombineable sourceCatalogueCombineable) + { + if (sourceCatalogueCombineable.Catalogue.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) + { + var dx = (DataExportChildProvider)_activator.CoreChildProvider; + var cic = targetJoinableCollectionNode.Configuration; + var cicProjAssociations = dx.AllProjectAssociatedCics.Where(c => c.CohortIdentificationConfiguration_ID == cic.ID).ToArray().Select(a => a.Project); + var extractableDatasets = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWithParent(sourceCatalogueCombineable.Catalogue).ToList(); + var catalogueProjects = extractableDatasets.SelectMany(e => e.Projects); + if (!catalogueProjects.Any(c => cicProjAssociations.Contains(c))) + { + return null; + } + + } + return new ExecuteCommandAddCatalogueToCohortIdentificationAsPatientIndexTable(_activator, + sourceCatalogueCombineable, targetJoinableCollectionNode.Configuration); + } + return (ICommandExecution)null; } From bf4bdb4481d07070d40c3751afb72810e1316b45 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 17 Nov 2025 08:58:04 +0000 Subject: [PATCH 132/142] update cic clone --- ...dCloneCohortIdentificationConfiguration.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneCohortIdentificationConfiguration.cs index 0eb139ad6f..11c062d005 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneCohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneCohortIdentificationConfiguration.cs @@ -8,11 +8,14 @@ using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.DataExport.Data; using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.Providers; using Rdmp.Core.Repositories.Construction; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; +using System; +using System.Linq; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -65,6 +68,17 @@ public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) return this; } + private bool Validate(out string reason) + { + if(_cic.RootCohortAggregateContainer.GetAllAggregateConfigurationsRecursively().Select(a => a.Catalogue).Where(c => c.IsInternalDataset).Any()) + { + reason = "Configuration contains Catalogues marked as Internal"; + return false; + } + reason = null; + return true; + } + public override void Execute() { base.Execute(); @@ -74,6 +88,12 @@ public override void Execute() if (_cic == null) return; + if (!Validate(out string reason)) + { + Show("Unable to clone Cohort Identification Configuration",$"Cohort Identification Configuration is not in a valid state to clone.{reason}."); + return; + } + // Confirm creating yes/no (assuming activator is interactive) if (!_autoConfirm && BasicActivator.IsInteractive && !YesNo( "This will create a 100% copy of the entire CohortIdentificationConfiguration including all datasets, filters, parameters and set operations. Are you sure this is what you want?", From ae31de5d5807df81de34ee1f62c84ca157db37d8 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 17 Nov 2025 09:36:07 +0000 Subject: [PATCH 133/142] cohort sets --- CHANGELOG.md | 1 + ...cutionWhenTargetIsCohortAggregateContainer.cs | 13 +++++++++++++ .../RDMPCommandExecutionFactory.cs | 16 +++++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7551acfcb5..e5b38d3b24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.0.4] - Unreleased - Fix bug with duplicate searchables - Introduce ability to view Catalogues in a flat view +- Fix bug where Internal catalogues were still able to be added to a CIC ## [9.0.3] - 2025-11-03 - Improve checking for default pipelines diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCohortAggregateContainer.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCohortAggregateContainer.cs index 6df0b4581f..9bc5dff18c 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCohortAggregateContainer.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCohortAggregateContainer.cs @@ -67,6 +67,19 @@ public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, sourceAggregateCommand, targetCohortAggregateContainer); case AggregateConfigurationCombineable sourceAggregateCommand: { + if (sourceAggregateCommand.Aggregate.Catalogue.IsProjectSpecific(ItemActivator.RepositoryLocator.DataExportRepository)) + { + var dx = (DataExportChildProvider)ItemActivator.CoreChildProvider; + var acic = targetCohortAggregateContainer.GetCohortIdentificationConfiguration(); + var cicProjAssociations = dx.AllProjectAssociatedCics.Where(c => c.CohortIdentificationConfiguration_ID == acic.ID).ToArray().Select(a => a.Project); + var extractableDatasets = ItemActivator.RepositoryLocator.DataExportRepository.GetAllObjectsWithParent(sourceAggregateCommand.Aggregate.Catalogue).ToList(); + var catalogueProjects = extractableDatasets.SelectMany(e => e.Projects); + if (!catalogueProjects.Any(c => cicProjAssociations.Contains(c))) + { + return null; + } + + } var cic = sourceAggregateCommand.CohortIdentificationConfigurationIfAny; if (cic != null && !cic.Equals(targetCohortAggregateContainer.GetCohortIdentificationConfiguration())) diff --git a/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs b/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs index b3df6ba4b7..720f6908cb 100644 --- a/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs +++ b/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs @@ -161,10 +161,24 @@ private ICommandExecution CreateWhenTargetIsJoinableCollectionNode(ICombineToMak JoinableCollectionNode targetJoinableCollectionNode) { if (cmd is AggregateConfigurationCombineable sourceAggregateConfigurationCombineable) + { + if (sourceAggregateConfigurationCombineable.Aggregate.Catalogue.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) + { + var dx = (DataExportChildProvider)_activator.CoreChildProvider; + var acic = targetJoinableCollectionNode.Configuration; + var cicProjAssociations = dx.AllProjectAssociatedCics.Where(c => c.CohortIdentificationConfiguration_ID == acic.ID).ToArray().Select(a => a.Project); + var extractableDatasets = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWithParent(sourceAggregateConfigurationCombineable.Aggregate.Catalogue).ToList(); + var catalogueProjects = extractableDatasets.SelectMany(e => e.Projects); + if (!catalogueProjects.Any(c => cicProjAssociations.Contains(c))) + { + return null; + } + + } if (sourceAggregateConfigurationCombineable.Aggregate.IsCohortIdentificationAggregate) return new ExecuteCommandConvertAggregateConfigurationToPatientIndexTable(_activator, sourceAggregateConfigurationCombineable, targetJoinableCollectionNode.Configuration); - + } if (cmd is CatalogueCombineable sourceCatalogueCombineable) { if (sourceCatalogueCombineable.Catalogue.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) From 830730b35f2e3b0f93550243b34a28ba12b7db3e Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 17 Nov 2025 14:35:25 +0000 Subject: [PATCH 134/142] codeql --- Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs b/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs index 720f6908cb..4e9440d7e9 100644 --- a/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs +++ b/Rdmp.UI/CommandExecution/RDMPCommandExecutionFactory.cs @@ -197,7 +197,7 @@ private ICommandExecution CreateWhenTargetIsJoinableCollectionNode(ICombineToMak return new ExecuteCommandAddCatalogueToCohortIdentificationAsPatientIndexTable(_activator, sourceCatalogueCombineable, targetJoinableCollectionNode.Configuration); } - return (ICommandExecution)null; + return null; } From 1464eea67a64474c44b73f8ecb93038dd66d9919 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 19 Nov 2025 07:56:50 +0000 Subject: [PATCH 135/142] Task/rdmp 343 Cross-Project Commits Flow (#2261) * add files * update cancel * fix cancel * moving cics * add projects column to cic tree * update text --- .../WindowManagement/ActivateItems.cs | 7 + CHANGELOG.md | 1 + ...utingACohortIdentificationConfiguration.cs | 28 +- .../CommandExecution/BasicActivateItems.cs | 8 +- .../CommandExecution/IBasicActivateItems.cs | 1 + ...hortIdentificationCollectionUI.Designer.cs | 10 +- .../CohortIdentificationCollectionUI.cs | 15 +- ...CohortCommitProjectSelectionUI.Designer.cs | 87 +++ .../Cohorts/CohortCommitProjectSelectionUI.cs | 130 ++++ .../CohortCommitProjectSelectionUI.resx | 559 ++++++++++++++++++ 10 files changed, 839 insertions(+), 7 deletions(-) create mode 100644 Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.resx diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs b/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs index f6f05a4427..0c2250c1b1 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs @@ -55,6 +55,7 @@ using Rdmp.UI.Refreshing; using Rdmp.UI.Rules; using Rdmp.UI.SimpleDialogs; +using Rdmp.UI.SimpleDialogs.Cohorts; using Rdmp.UI.SimpleDialogs.ForwardEngineering; using Rdmp.UI.SingleControlForms; using Rdmp.UI.SubComponents; @@ -904,6 +905,12 @@ public override CohortCreationRequest GetCohortCreationRequest(ExternalCohortTab return ui.ShowDialog() == DialogResult.OK ? ui.Result : null; } + public override IProject CohortCommitProjectSelect(IProject currentProject,Project[] projects) + { + var ui = new CohortCommitProjectSelectionUI(this, currentProject, projects); + return ui.ShowDialog() == DialogResult.OK ? ui.Result : null; ; + } + public override CohortHoldoutLookupRequest GetCohortHoldoutLookupRequest(ExternalCohortTable externalCohortTable, IProject project, CohortIdentificationConfiguration cic) { diff --git a/CHANGELOG.md b/CHANGELOG.md index cead152b81..19a9e6f65f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.1.0] - Unreleased - Fix bug with duplicate searchables +- Improve UI for committing cohorts across projects - Add the ability to Template Cohort Identification Configurations (see [Documentation\cohorts\CohortIdentificationConfigurationTemplates.md]) - Require all deletes to enter a commit message when using the commit system - Introduce ability to view Catalogues in a flat view diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs index 39dba10eb1..8e92bf6c98 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs @@ -83,11 +83,19 @@ public override void Execute() } } - if (Project == null && BasicActivator.CoreChildProvider is DataExportChildProvider dx) + + IProject currentProj = null; + ProjectCohortIdentificationConfigurationAssociation[] projAssociations = Array.Empty(); + if (BasicActivator.CoreChildProvider is DataExportChildProvider dx) { - var projAssociations = dx.AllProjectAssociatedCics + projAssociations = dx.AllProjectAssociatedCics .Where(c => c.CohortIdentificationConfiguration_ID == cic.ID).ToArray(); - if (projAssociations.Length == 1) Project = projAssociations[0].Project; + if (projAssociations.Length > 0) + { + currentProj = Project != null ? Project : projAssociations.Length == 1 ? projAssociations[0].Project : null; + Project = BasicActivator.CohortCommitProjectSelect(currentProj, BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjects().ToArray()); + if (Project is null) return; + } } var auditLogBuilder = new ExtractableCohortAuditLogBuilder(); @@ -95,7 +103,10 @@ public override void Execute() //user choose to cancel the cohort creation request dialogue if (request == null) + { + Project = null; return; + } request.CohortIdentificationConfiguration = cic; @@ -103,7 +114,16 @@ public override void Execute() configureAndExecute.PipelineExecutionFinishedsuccessfully += (s, u) => OnImportCompletedSuccessfully(cic); - configureAndExecute.Run(BasicActivator.RepositoryLocator, null, null, null); + var result = configureAndExecute.Run(BasicActivator.RepositoryLocator, null, null, null); + if (result ==0 && currentProj != null && currentProj.ID != Project.ID) + { + //move cic to new project + var oldAssociation = projAssociations.Where(a => a.Project_ID == currentProj.ID).FirstOrDefault(); + if (oldAssociation != null) + { + oldAssociation.DeleteInDatabase(); + } + } } private void OnImportCompletedSuccessfully(CohortIdentificationConfiguration cic) diff --git a/Rdmp.Core/CommandExecution/BasicActivateItems.cs b/Rdmp.Core/CommandExecution/BasicActivateItems.cs index 84cf904b72..e0583256f6 100644 --- a/Rdmp.Core/CommandExecution/BasicActivateItems.cs +++ b/Rdmp.Core/CommandExecution/BasicActivateItems.cs @@ -666,6 +666,12 @@ public virtual CohortCreationRequest GetCohortCreationRequest(ExternalCohortTabl RepositoryLocator.DataExportRepository, cohortInitialDescription); } + public virtual IProject CohortCommitProjectSelect(IProject currentProject, Project[] projects) + { + return currentProject; + } + + /// public virtual CohortHoldoutLookupRequest GetCohortHoldoutLookupRequest(ExternalCohortTable externalCohortTable, IProject project, CohortIdentificationConfiguration cic) { @@ -673,7 +679,7 @@ public virtual CohortHoldoutLookupRequest GetCohortHoldoutLookupRequest(External if (!TypeText("Name", "Enter name for cohort", 255, null, out var name, false)) throw new Exception("User chose not to enter a name for the cohort and none was provided"); - return new CohortHoldoutLookupRequest(cic, "empty", 1,false,"",""); + return new CohortHoldoutLookupRequest(cic, "empty", 1, false, "", ""); } /// diff --git a/Rdmp.Core/CommandExecution/IBasicActivateItems.cs b/Rdmp.Core/CommandExecution/IBasicActivateItems.cs index 6c84bc6988..afe089976b 100644 --- a/Rdmp.Core/CommandExecution/IBasicActivateItems.cs +++ b/Rdmp.Core/CommandExecution/IBasicActivateItems.cs @@ -173,6 +173,7 @@ public interface IBasicActivateItems CohortCreationRequest GetCohortCreationRequest(ExternalCohortTable externalCohortTable, IProject project, string cohortInitialDescription); + IProject CohortCommitProjectSelect(IProject currentProject, Project[] projects); CohortHoldoutLookupRequest GetCohortHoldoutLookupRequest(ExternalCohortTable externalCohortTable, IProject project, CohortIdentificationConfiguration cic); diff --git a/Rdmp.UI/Collections/CohortIdentificationCollectionUI.Designer.cs b/Rdmp.UI/Collections/CohortIdentificationCollectionUI.Designer.cs index f2043e27ba..3b49263b89 100644 --- a/Rdmp.UI/Collections/CohortIdentificationCollectionUI.Designer.cs +++ b/Rdmp.UI/Collections/CohortIdentificationCollectionUI.Designer.cs @@ -34,6 +34,7 @@ private void InitializeComponent() components = new System.ComponentModel.Container(); tlvCohortIdentificationConfigurations = new TreeListView(); olvName = new OLVColumn(); + olvAssociatedProjects = new OLVColumn(); olvFrozen = new OLVColumn(); tbFilter = new System.Windows.Forms.TextBox(); ((System.ComponentModel.ISupportInitialize)tlvCohortIdentificationConfigurations).BeginInit(); @@ -42,9 +43,10 @@ private void InitializeComponent() // tlvCohortIdentificationConfigurations // tlvCohortIdentificationConfigurations.AllColumns.Add(olvName); + tlvCohortIdentificationConfigurations.AllColumns.Add(olvAssociatedProjects); tlvCohortIdentificationConfigurations.AllColumns.Add(olvFrozen); tlvCohortIdentificationConfigurations.CellEditUseWholeCell = false; - tlvCohortIdentificationConfigurations.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] { olvName, olvFrozen }); + tlvCohortIdentificationConfigurations.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] { olvName, olvAssociatedProjects,olvFrozen }); tlvCohortIdentificationConfigurations.Dock = System.Windows.Forms.DockStyle.Fill; tlvCohortIdentificationConfigurations.Location = new System.Drawing.Point(0, 0); tlvCohortIdentificationConfigurations.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); @@ -63,6 +65,11 @@ private void InitializeComponent() olvName.Sortable = false; olvName.Text = "Cohort Identification Configurations"; olvName.Width = 100; + // + // olvAssociatedProjects + // + olvAssociatedProjects.IsEditable = false; + olvAssociatedProjects.Text = "Associated Projects"; // // olvFrozen // @@ -95,6 +102,7 @@ private void InitializeComponent() private TreeListView tlvCohortIdentificationConfigurations; private OLVColumn olvName; + private OLVColumn olvAssociatedProjects; private OLVColumn olvFrozen; private System.Windows.Forms.TextBox tbFilter; } diff --git a/Rdmp.UI/Collections/CohortIdentificationCollectionUI.cs b/Rdmp.UI/Collections/CohortIdentificationCollectionUI.cs index 3d03f2f1e8..7b61631bc2 100644 --- a/Rdmp.UI/Collections/CohortIdentificationCollectionUI.cs +++ b/Rdmp.UI/Collections/CohortIdentificationCollectionUI.cs @@ -13,6 +13,7 @@ using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.Providers; using Rdmp.Core.Providers.Nodes.CohortNodes; using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.CommandExecution.AtomicCommands.UIFactory; @@ -37,6 +38,7 @@ public CohortIdentificationCollectionUI() { InitializeComponent(); olvFrozen.AspectGetter = FrozenAspectGetter; + olvAssociatedProjects.AspectGetter = AssociatedProjectsAspectGetter; } public override void SetItemActivator(IActivateItems activator) @@ -114,6 +116,7 @@ public override void SetItemActivator(IActivateItems activator) if (_firstTime) { CommonTreeFunctionality.SetupColumnTracking(olvName, new Guid("f8a42259-ce5a-4006-8ab8-e0305fce05aa")); + CommonTreeFunctionality.SetupColumnTracking(olvAssociatedProjects, new Guid("f8a42259-ce5a-4006-8ab8-e0305fce05aa")); CommonTreeFunctionality.SetupColumnTracking(olvFrozen, new Guid("d1e155ef-a28f-41b5-81e4-b763627ddb3c")); tlvCohortIdentificationConfigurations.Expand(rootFolder); @@ -148,7 +151,17 @@ root is FolderNode f public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) { } - + private string AssociatedProjectsAspectGetter(object o) + { + var cic = o as CohortIdentificationConfiguration; + if (cic != null) + { + var dx = Activator.CoreChildProvider as DataExportChildProvider; + var associations = dx.AllProjectAssociatedCics.Where(c => c.CohortIdentificationConfiguration_ID == cic.ID); + return string.Join(", ", associations.Select(a => a.Project.ID)); + } + return ""; + } private string FrozenAspectGetter(object o) => o is CohortIdentificationConfiguration cic ? cic.Frozen ? "Yes" : "No" : null; } \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.Designer.cs new file mode 100644 index 0000000000..5ba996b649 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.Designer.cs @@ -0,0 +1,87 @@ +namespace Rdmp.UI.SimpleDialogs.Cohorts +{ + partial class CohortCommitProjectSelectionUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + System.ComponentModel.ComponentResourceManager resources = new System.ComponentModel.ComponentResourceManager(typeof(CohortCommitProjectSelectionUI)); + btnCurrentProject = new System.Windows.Forms.Button(); + btnNewProject = new System.Windows.Forms.Button(); + btnExistingProject = new System.Windows.Forms.Button(); + SuspendLayout(); + // + // btnCurrentProject + // + btnCurrentProject.Location = new System.Drawing.Point(12, 12); + btnCurrentProject.Name = "btnCurrentProject"; + btnCurrentProject.Size = new System.Drawing.Size(187, 23); + btnCurrentProject.TabIndex = 0; + btnCurrentProject.Text = "This Project"; + btnCurrentProject.UseVisualStyleBackColor = true; + btnCurrentProject.Click += btnCurrentProject_Click; + // + // btnNewProject + // + btnNewProject.Location = new System.Drawing.Point(12, 41); + btnNewProject.Name = "btnNewProject"; + btnNewProject.Size = new System.Drawing.Size(187, 23); + btnNewProject.TabIndex = 1; + btnNewProject.Text = "A New Project"; + btnNewProject.UseVisualStyleBackColor = true; + btnNewProject.Click += btnNewProject_Click; + // + // btnExistingProject + // + btnExistingProject.Location = new System.Drawing.Point(12, 70); + btnExistingProject.Name = "btnExistingProject"; + btnExistingProject.Size = new System.Drawing.Size(187, 23); + btnExistingProject.TabIndex = 2; + btnExistingProject.Text = "An Existing Project"; + btnExistingProject.UseVisualStyleBackColor = true; + btnExistingProject.Click += btnExistingProject_Click; + // + // CohortCommitProjectSelectionUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(238, 194); + Controls.Add(btnExistingProject); + Controls.Add(btnNewProject); + Controls.Add(btnCurrentProject); + Icon = (System.Drawing.Icon)resources.GetObject("$this.Icon"); + Name = "CohortCommitProjectSelectionUI"; + Text = "Commit Cohort"; + ResumeLayout(false); + } + + #endregion + + private System.Windows.Forms.Button btnCurrentProject; + private System.Windows.Forms.Button btnNewProject; + private System.Windows.Forms.Button btnExistingProject; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.cs b/Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.cs new file mode 100644 index 0000000000..8662422f5d --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.cs @@ -0,0 +1,130 @@ +using NPOI.SS.Formula.Functions; +using Rdmp.Core.CohortCommitting.Pipeline; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.Providers; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Cohorts +{ + public partial class CohortCommitProjectSelectionUI : RDMPForm + { + private readonly IProject _currentProject; + private readonly Project[] _projects; + private readonly IActivateItems _activator; + public CohortCommitProjectSelectionUI(IActivateItems activator, IProject currentProject, Project[] projects) + { + InitializeComponent(); + _activator = activator; + _currentProject = currentProject; + _projects = projects; + if (_currentProject != null) + { + btnCurrentProject.Text = $"This Project ({_currentProject.Name.Substring(0,Math.Min(10,_currentProject.Name.Length))}{(_currentProject.Name.Length>0?"...":"")})"; + } + else + { + btnCurrentProject.Enabled = false; + } + } + + [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] + public IProject Result { get; set; } + + private void btnCurrentProject_Click(object sender, EventArgs e) + { + Result = _currentProject; + DialogResult = DialogResult.OK; + Close(); + } + + private void btnNewProject_Click(object sender, EventArgs e) + { + var p = new ProjectUI.ProjectUI(); + var dialog = new RDMPForm(_activator); + + p.SwitchToCutDownUIMode(); + p.SetItemActivator(_activator); + var ok = new Button(); + ok.Click += (s, ev) => + { + dialog.Close(); + dialog.DialogResult = DialogResult.OK; + }; + ok.Location = new Point(0, p.Height + 10); + ok.Width = p.Width / 2; + ok.Height = 30; + ok.Text = "Ok"; + + var cancel = new Button(); + cancel.Click += (s, ev) => + { + dialog.Close(); + dialog.DialogResult = DialogResult.Cancel; + }; + cancel.Location = new Point(p.Width / 2, p.Height + 10); + cancel.Width = p.Width / 2; + cancel.Height = 30; + cancel.Text = "Cancel"; + + dialog.Controls.Add(ok); + dialog.Controls.Add(cancel); + + dialog.Height = p.Height + 80; + dialog.Width = p.Width + 10; + dialog.Controls.Add(p); + + ok.Anchor = AnchorStyles.Bottom; + cancel.Anchor = AnchorStyles.Bottom; + + var project = new Project(_activator.RepositoryLocator.DataExportRepository, "New Project"); + p.SetDatabaseObject(_activator, project); + var result = dialog.ShowDialog(); + if (result == DialogResult.OK) + { + project.SaveToDatabase(); + if (_currentProject != null) + { + var projectSpecificCatalogues = _currentProject.GetAllProjectCatalogues().Where(p => p.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)); + foreach (var psc in projectSpecificCatalogues) + { + var cmd = new ExecuteCommandMakeCatalogueProjectSpecific(_activator, psc, project, true); + cmd.Execute(); + } + } + + _activator.Publish(project); + DialogResult = DialogResult.OK; + Result = project; + Close(); + } + } + private void btnExistingProject_Click(object sender, EventArgs e) + { + var selected = _activator.SelectOne(new DialogArgs + { + TaskDescription = + "Choose a Project which this cohort will be associated with. This will set the cohorts ProjectNumber. A cohort can only be extracted from a Project whose ProjectNumber matches the cohort (multiple Projects are allowed to have the same ProjectNumber)" + }, _projects); + if (selected != null) + { + Result = selected as Project; + DialogResult = DialogResult.OK; + Close(); + } + } + } +} diff --git a/Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.resx b/Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.resx new file mode 100644 index 0000000000..0c40708b3b --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Cohorts/CohortCommitProjectSelectionUI.resx @@ -0,0 +1,559 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + + AAABAAUAQEAAAAEAIAAoQgAAVgAAACAgAAABACAAqBAAAH5CAAAgIAAAAQAIAKgIAAAmUwAAEBAAAAEA + IABoBAAAzlsAABAQAAABAAgAaAUAADZgAAAoAAAAQAAAAIAAAAABACAAAAAAAACAAAAAAAAAAAAAAAAA + AAAAAAAA//////39/f/29vb/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/29vb//f39///////39/f/xcXF/7Kysv+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gx + sf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gx + sf+xsbH/sbGx/7CwsP+wsLD/sLCw/7CwsP+wsLD/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gx + sf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gx + sf+xsbH/sbGx/7CwsP+wsLD/sLCw/7CwsP+xsbH/xMTE//f39///////9fX1/9vb2//39/f/9vb2//b2 + 9v/29vb/+fn5//r6+v/6+vr/+vr6//f39//39/f/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+fn5//b2 + 9v/39/f/9/f3//f39//39/f/9/f3//f39//4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/39/f/9/f3//j4 + +P/4+Pj/+fn5//n5+f/6+vr/+vr6//f39//39/f/9/f3//f39//4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4 + +P/39/f/9/f3//f39//39/f/9/f3//f39//39/f/9/f3//f39//39/f/9/f3/9jY2P/09PT///////X1 + 9f/b29v///////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + ///a2tr/8/Pz///////19fX/29vb////////////9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T0 + 9P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T0 + 9P/09PT/9PT0//T09P/09PT/8/Pz//T09P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T0 + 9P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T0 + 9P/09PT/9PT0//7+/v//////2tra//Pz8///////9fX1/9vb2////////////+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//ycnk/9XV6P/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9ra2v/z8/P///////X19f/c3Nz///////// + ///v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+vp4f+/p0b/spMX/7GTFv++pkP/6+ni/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/8fH4//T0+j/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////a2tr/8/Pz//// + ///19fX/2tra////////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/9bJl/+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv/Ux5D/7+/v/+/v7//v7+//7+/v/+/v7//FxeP/0dHm/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+ + /v//////2NjY//Pz8///////9fX1/9XV1f///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+Pd + xP+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/+bi0f/v7+//7+/v/+/v7//v7+//w8Pi/9DQ + 5v/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//+/v7//////9jY2P/z8/P///////X19f/V1dX////////////v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7/+4nC3/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+wkRL/7+/v/+/v + 7//v7+//7+/v/8DA4v/OzuX/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////Y2Nj/8/Pz///////19fX/1dXV//// + ////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/+7t6//v7+//7+/v/+/v7/+/v+H/zc3l/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2NjY//Pz + 8///////9fX1/9XV1f///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv/t7Oj/7+/v/+/v7//v7+//vLzh/8zM5f/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//+/v7//////9nZ2f/z8/P///////X19f/S0tL////////////v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+DY + uv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/493F/+/v7//v7+//7+/v/7q6 + 4P/KyuT/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+///v7+///////X19f/8/Pz///////19fX/0tLS////////////7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//6eba/7mdL//ZzqH/zr17/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/zLt3/9PG + j/+7oDX/6+nh/+/v7/+4uN//ycnk/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////19fX//Pz8///////9fX1/9HR + 0f///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+7/w61V/8i1af/u7ev/7+/v/+7u7f+zlRr/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/tZch/+7u7v/v7+//7e3p/8SvWf/HtGP/tbXd/8bG4v/v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9bW + 1v/z8/P///////X19f/R0dH////////////v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+7u7f/Ux5P/u6E4/+rn3v/v7+//7+/v/+/v7//v7+//7+/u/8Ot + Vf+tjAj/rYwG/62MBv+tjQj/wqtR/+7u7f/v7+//7+/v/+/v7//v7+//6OXZ/7GXMf/Gu53/7u7u/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+///v7+///////W1tb/8/Pz///////19fX/0dHR////////////7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//4tvA/7idL/+wkBH/tJYd/9fMnf/v7+7/7+/v/+PeyP+ykxn/5N/K/+7u7v/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7u3s/+vp4f/r6eD/7u3r/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7/+wsN3/ubK1/7KUG//o5Nb/7+/v/+7u7v/YzZ//tJYf/7CQEf+3myv/4Nq+/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////19fX//Pz8///////9fX1/8/Pz////////////+/v + 7//v7+//7+/v/+/v7//u7u7/u6I5/62MBv+tjAb/rYwG/62MBv+tjAb/spQa/7ugNv/Ux5H/7u7u/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//ra3c/8PD4v/u7u3/0MGD/76mRf+xkxb/rYwG/62MBv+tjAb/rYwG/62M + Bv+6oDb/7u7t/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9fX1//z8/P///////X1 + 9f/Pz8/////////////v7+//7+/v/+/v7//v7+//wqtQ/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+wkRL/7+/u/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/6mp2//BweH/7+/v/+7u7v+0lh//rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/8GqTf/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+//// + ///X19f/8/Pz///////19fX/z8/P////////////7+/v/+/v7//v7+//6+nh/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/93Vsv/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+np9r/v7/h/+/v + 7//h2rz/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/7Ovm/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v//7+/v//////19fX//Pz8///////9fX1/8/Pz////////////+/v7//v7+//7+/v/9rQ + pv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv/DrVP/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//pKTZ/76+4f/v7+//vqZD/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/9rP + pv/v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9fX1//z8/P///////X19f/Ozs7///////// + ///v7+//7+/v/+/v7//WyZX/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/vaRA/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/6Cg2P+8vOD/7+/v/7yiOv+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv/VyJP/7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////X19f/8/Pz//// + ///19fX/0NDQ////////////7+/v/+/v7//v7+//493F/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/829ev/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+dndj/w8Pi/+/v7//Pv37/rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/5eDL/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+ + /v//////2dnZ//Pz8///////9fX1/8/Pz////////////+/v7//v7+//7+/v/+7u7v+vjw//rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+Cf8D/7Ozu/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//lJTV/7q6 + 4P/v7+//7evn/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/r48O/+/v7v/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//+/v7//////9nZ2f/z8/P///////X19f/Ozs7////////////v7+//7+/v/+/v + 7//v7+//5N3H/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv/RwYT/29vp/yEhtP+vr93/7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/5eX1v+4uN//7+/v/+/v7//QwYP/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/+Pc + w//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////a2tr/8/Pz///////19fX/zs7O//// + ////////7+/v/+/v7//v7+//7+/v/+/v7//j3cX/s5Qa/62MBv+tjAb/rYwG/6+PDf/WyZf/7+/u/+/v + 7//v7+//o6PZ/ysruP/KyuT/7+/v/+/v7//v7+//7+/v/+7t6//t7er/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+UlNX/t7ff/+/v7//v7+//7+/v/9jMnP+vjw7/rYwG/62M + Bv+tjAb/spQa/+Lbwf/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2tra//Pz + 8///////9fX1/8/Pz////////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//t7Or/s5Qb/+zq + 5P/u7u7/7+/v/+/v7//v7+//7+/v/+/v7//u7u7/ZWXH/z4+vf/p6e3/7u7s/7icLv+tjAb/rYwG/7GT + F//q6N7/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//kZHU/7W13//v7+//7+/v/+/v + 7//v7+//7u7u/+vq5P+7oDf/7ezp/+7u7v/v7+//7+/v/+/v7//v7+//7+/v/+/v7//q6u7/Tk7B/+/v + 7//+/v7//////9ra2v/z8/P///////X19f/S0tL////////////v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/7SWH//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//g4Or/MjK4/6qL + FP+tjAb/rYwG/62MBv+tjAb/r48N/+7u7v/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/46O + 1P+zs97/7+/v/+/v7//v7+//7+/v/+/v7//u7u7/vaM+/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+3t + 7v9+fs7/LCy3/9nZ6f/v7+///v7+///////Z2dn/8/Pz///////19fX/0tLS////////////7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+0lh//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+Lbw/+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv/VyZX/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7/+Li9L/srLd/+/v7//v7+//7+/v/+/v7//v7+//7u7u/72jPv/v7+//7+/v/+/v + 7//v7+//7+/v/7Cw3f8REa//urrg/+7u7v/v7+//7+/v//7+/v//////2NjY//Pz8///////9fX1/9LS + 0v///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//tJYf/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//ZzaH/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/w65W/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//h4fR/7Gx3f/v7+//7+/v/+/v7//v7+//7+/v/+7u + 7v+9oz7/7+/v/+/v7//v7+//zMzk/ycntv94eM3/7u7v/+/v7//v7+//7+/v/+/v7//+/v7//////9jY + 2P/z8/P///////T09P/S0tL////////////v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/7SW + H//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//5eHO/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/9fNpv/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/4SE0P+wsN3/7+/v/+/v + 7//v7+//7+/v/+/v7//u7u7/vaM+/+/v7//q6u3/Ojq7/zQ0uv/t7e//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+///v7+///////X19f/8/Pz///////19fX/0tLS////////////7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7/+0lh//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7v+3myz/rYwG/62MBv+tjAb/rYwG/7CSGf8VFbH/dHTM/+3t7v/v7+//7+/v/+/v7//v7+//7+/v/+/v + 7/+AgM//rq7c/+/v7//v7+//7+/v/+/v7//v7+//7u7u/7mfPf9kZMj/ICC0/8bG4//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2NjY//Pz8///////9fX1/9LS0v///////////+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//tJYf/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/8i1Zv+tjAf/rYwG/7+mRv/u7ev/7+/v/5mZ1/8JCa//ra3c/+7u + 7//v7+//7+/v/+/v7//v7+//fHzO/62t3P/v7+//7+/v/+/v7//v7+//7u7u/6Wl2f+HbS7/nJzX/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9nZ2f/z8/P///////X1 + 9f/T09P////////////v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/7SWH//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//6uru/01Nwf8gILT/y8vl/+/v7//v7+//7+/v/3h4zf+rq9z/7+/v/+/v7//v7+//xsbj/xwc + s/9aWsT/vaM+/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+//// + ///Z2dn/8/Pz///////19fX/09PT////////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7/+0lh//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//5eXs/xQUsf82Nrr/7Ozu/+/v7/90dMz/qqrc/+/v + 7//q6u7/Kyu3/xkZsv/k5Oz/7u7u/72jPv/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v//7+/v//////2dnZ//Pz8///////9fX1/9LS0v///////////+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//tJYf/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//u7u//n5/Y/w0N + r/9zc8v/bm7K/6Wl2v9hYcf/ERGw/7Gx3f/v7+//7+/v/+7u7v+9oz7/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9nZ2f/z8/P///////X19f/S0tL///////// + ///v7+//7+/v/+/v7//v7+//7+/v/+7u7v/n5NX/xK9a/66OCv+/qEn/4dq9/+7u7v/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//t7e//WFjE/wYGrf8GBq3/eXnN/+/v7//v7+//7u7u/76+4f9JScD/JiGa/1hY + xP/T0+f/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////Z2dn/8/Pz//// + ///19fX/0NDQ////////////7+/v/+/v7//v7+//7+/v/+/v7v/Cq1H/rYwG/62MBv+tjAb/rYwG/62M + Bv+2mSb/7e3q/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//u7u//yMjk/xYWsf8TE7D/Ghqy/ykptv/a2un/6Ojt/yIi + tP8GBq3/Bgat/wYGrf8GBq3/Bgat/05Owf/u7u7/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+ + /v//////1tbW//Pz8///////9fX1/8/Pz////////////+/v7//v7+//7+/v/+/v7//Hs2P/rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/7KTGP/u7u7/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//p6e3/NTW5/xoasv/b2+n/b2/L/6qq + 3P/IyOT/Dg6v/w8PsP8GBq3/Bgat/wYGrf8GBq3/Bgat/wYGrf8GBq3/X1/G/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//+/v7//////9fX1//z8/P///////X19f/Pz8/////////////v7+//7+/v/+/v + 7//s6uT/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/4Ni7/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+zs7v93d83/FBSx/6ys + 3P/v7+//7+/v/3Fxy/+trdz/7+/v/8XF4/8GBq3/Bgat/wYGrf8GBq3/Bgat/wYGrf8GBq3/Bgat/wYG + rf/o6O3/7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////X19f/8/Pz///////19fX/z8/P//// + ////////7+/v/+/v7//v7+//3NKr/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/8Sv + Wf/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7u7u/7q6 + 3/8MDK7/fX3O/+/v7//v7+//7+/v/+/v7/9zc8v/sLDd/+/v7/9ZWcT/Bgat/wYGrf8GBq3/Bgat/wYG + rf8GBq3/Bgat/wYGrf8GBq3/rKzc/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////19fX//Pz + 8///////9fX1/87Ozv///////////+/v7//v7+//7+/v/9XHlP+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv+9pD7/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//3d3p/yYmtv9SUsL/5+fs/+/v7//v7+//7+/v/+/v7//v7+//dXXM/7Gx3f/v7+//OTm7/wYG + rf8GBq3/Bgat/wYGrf8GBq3/Bgat/wYGrf8GBq3/Bgat/5GR1P/v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//+/v7//////9jY2P/z8/P///////X19f/Ozs7////////////v7+//7+/v/+/v7//i2r//rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/wahL/+7u7f/v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//s7O7/V1fD/yQktf/e3ur/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/3d3 + zP+0tN//7u7t/2ZfmP8GBq3/Bgat/wYGrf8GBq3/Bgat/wYGrf8GBq3/Bgat/wYGrf/ExOL/7+/v/+/v + 7//v7+//7+/v/+/v7//v7+///v7+///////Y2Nj/8/Pz///////19fX/zs7O////////////7+/v/+/v + 7//v7+//7u7s/6+ODP+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/97Wtf+3nC7/3NKr/+/u + 7v/v7+//7+/v/+/v7//v7+//7+/v/+/v7/+Tk9X/Ghqy/7e33//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7/95ec3/qJ2S/7qfNf/a1tP/Bgat/wYGrf8GBq3/Bgat/wYGrf8GBq3/Bgat/wYG + rf8GBq3/vb3g/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2dnZ//Pz8///////9fX1/8/P + z////////////+/v7//v7+//7+/v/+/v7//d1LD/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/8az + Yv/v7+//7+/v/9vRqP+0lyD/6eba/+/u7v/v7+//7+/v/87O5v8REbD/j4/U/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7v/n5Nb/rI4d/8O8sv/v7+//7+/v/2pqyf8GBq3/Bgat/wYG + rf8GBq3/Bgat/wYGrf8GBq3/rKzc/1NTwv82Nrr/5eXs/+/v7//v7+//7+/v/+/v7//+/v7//////9jY + 2P/z8/P///////X19f/Pz8/////////////v7+//7+/v/+/v7//v7+//7+/v/97Wtf+ujQr/rYwG/62M + Bv+tjAb/rYwG/9DAgf/u7u7/7+/v/+/v7//v7+//7u3s/8y6c//BqUz/5OTo/zo6vP9lZcf/6ent/+7t + 7P/Swof/vKQ+/72kQP/SxIz/7+7t/+/v7//v7+//7+/v/+3s6P+9pEH/0cGG/3t7zP+8vOD/7+/v/+/v + 7//u7u//hobR/wYGrf8GBq3/Bgat/wYGrf8KCq7/srLe/+/v7//v7+//2trp/4qK0v/v7+//7+/v/+/v + 7//v7+///v7+///////Y2Nj/8/Pz///////19fX/z8/P////////////7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+XgzP/b0Kj/4tvB/+7u7v/v7+//7+/v/+/v7//v7+//7+/v/+/v7//t7e7/dXTD/56E + Mv/Huoj/7u7t/+DZvP+tjAf/rYwG/62MBv+tjAb/rYwG/62MCP/j3sf/7u7t/828eP++pkX/7ezp/+/v + 7/9+fs//v7/h/+/v7//v7+//7+/v/+/v7//u7u//wsLi/6io2//Ly+T/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2NjY//Pz8///////9fX1/9HR0f///////////+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7/+vr93/KSm3/8bG4//v7+//5N3G/7eaKf+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rowJ/7WZ + Jf/l4c7/7+/v/+/v7//v7+//gIDP/8LC4v/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9nZ2f/z8/P///////X1 + 9f/R0dH////////////v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+Dg6/8kJLX/pKTZ/+/v7//v7+//7+/v/+/v7/+4nS//rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv+7nzX/7+/v/+/v7//v7+//7+/v/4KC0P/ExOL/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+//// + ///Z2dn/8/Pz///////19fX/0dHR////////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7e3u/1dXxP92ds3/6urt/+/v7//v7+//7+/v/+/v7//v7+//rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/+/v7//v7+//7+/v/+/v7/+EhNH/yMjj/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v//7+/v//////2dnZ//Pz8///////9fX1/9HR0f///////////+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//u7u7/l5fW/0FBvv/m5uz/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv/u7ez/7+/v/+/v + 7//v7+//hobR/8rK5f/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9ra2v/z8/P///////X19f/Q0ND///////// + ///v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//Hx+P/MzO6/9HR5v/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+tjQn/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAf/7+/v/+/v7//v7+//7+/v/4iI0f/NzeX/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////a2tr/8/Pz//// + ///19fX/1dXV////////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+3t7v89Pbz/srLe/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//wqtQ/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/yLVo/+/v7//v7+//7+/v/+/v7/+KitL/0NDl/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+ + /v//////2tra//Pz8///////9fX1/9XV1f///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7e3u/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+zq + 5f+ujQr/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/sZIW/+/u7v/v7+//7+/v/+/v7//v7+//i4vS/9PT + 5//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//+/v7//////9ra2v/z8/P///////X19f/V1dX////////////v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7Ovn/7WXIv+tjAb/rYwG/62MBv+tjAb/tpop/+3s6P/v7+//7+/v/+/v + 7//v7+//7+/v/42N0//W1uj/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////a2tr/8/Pz///////19fX/1dXV//// + ////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7u7/6OXY/9rQpv/b0an/6efc/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+Pj9T/2dno/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2tra//Pz + 8///////9fX1/9XV1f///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//kJDU/9zc6v/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//+/v7//////9ra2v/z8/P///////X19f/b29v////////////v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/5SU + 1f/h4ev/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+///v7+///////a2tr/8/Pz///////19fX/29vb////////////7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2dnZ//Pz8///////9fX1/9vb + 2/////////////X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X1 + 9f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X1 + 9f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X1 + 9f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/+/v7//////9nZ + 2f/z8/P///////X19f/b29v///////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + ///////////////////Y2Nj/9PT0///////4+Pj/29vb//////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////2NjY//f39////////v7+//T09P/x8fH/8fHx//Hx + 8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx + 8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx + 8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx + 8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Pz8//+/v7///////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + /////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAACAAAABAAAAAAQAgAAAA + AAAAIAAAAAAAAAAAAAAAAAAAAAAAAPz8/P/X19f/0dHR/9HR0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR + 0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR + 0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR0f/r6+v/+vr6/+vr6//6+vr/+/v7//z8/P/7+/v/+/v7//v7 + +//7+/v/+/v7//r6+v/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//z8/P/7+/v/+/v7//v7 + +//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+bm5v/6+vr/7e3t//j4+P/x8fH/8fHx//Hx + 8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx/+Hh + 7f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/+/v7/5ubm//r6+v/s7Oz/9/f3/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//o5dn/wapM/6+PDv/AqUz/6OXX/+/v + 7//v7+//zMzk/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v/l5eX/+vr6/+rq + 6v/39/f/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/72kP/+tjAb/rYwG/62M + Bv+7ojv/7+/v/+/v7//IyOP/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+/+Xl + 5f/6+vr/6urq//f39//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//rYwG/62M + Bv+tjAb/rYwG/62MBv/u7ez/7+/v/8XF4//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//+/v7/5eXl//r6+v/o6Oj/9/f3/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7ezp/9jM + nv+1mCP/rYwG/62MBv+tjAb/tJci/9jMnv/u7ev/wcHh/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v//7+/v/l5eX/+vr6/+jo6P/39/f/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7u7u/9vR + qf/YzqL/7u7u/9/YuP+ylBr/rYwG/7KUGf/f2Lr/7u7t/9jNof+8s6P/7u7u/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+///v7+/+Tk5P/6+vr/5+fn//f39//v7+//7u7u/86+e/+wkRP/uZ4x/9LE + i//VyZf/7Orl/+/v7//v7+//7+/v/+7u7v/t7Of/7u7u/+/v7//v7+//7+/v/7a01P/WyZj/0sWN/7me + Mv+wkBL/zb16/+7u7v/v7+//7+/v/+/v7//+/v7/5eXl//r6+v/n5+f/9/f3/+/v7//Sw4n/rYwG/62M + Bv+tjAb/rY0J/+ro3//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//tLTd/+vp + 4f+ujgz/rYwG/62MBv+tjAb/0sOK/+/v7//v7+//7+/v//7+/v/l5eX/+vr6/+bm5v/39/f/7+/v/8Ks + Uf+tjAb/rYwG/62MBv+tjAb/18uc/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7/+vr9z/1cmW/62MBv+tjAb/rYwG/62MBv/Cq1H/7+/v/+/v7//v7+///v7+/+Xl5f/6+vr/5+fn//f3 + 9//v7+//y7lx/62MBv+tjAb/rYwG/62MBv/KxcX/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/6ur2//m4tD/rYwG/62MBv+tjAb/rYwG/8y6c//v7+//7+/v/+/v7//+/v7/5ubm//r6 + +v/m5ub/9/f3/+/v7//s6uX/vKI6/62MBv+tjAf/0MGD/7a23v+bm9f/5eXs/+/v7//u7u7/7u7t/+/v + 7//v7+//7+/v/+/v7//v7+//pqba/+/v7//QwYT/rYwI/62MBv+7oTn/7Ork/+/v7//v7+//7+/v//7+ + /v/m5ub/+vr6/+fn5//39/f/7+/v/+/v7//v7+//0MGE/+7t7P/v7+//7+/v/+7u7v+cnNf/rKSp/6+Q + EP+ujQr/3dSx/+/v7//v7+//7+/v/+/v7/+hodn/7+/v/+/v7//t7ev/1ceT/+7u7v/v7+//7u7u/6Cg + 2P/BweL//v7+/+bm5v/6+vr/6Ojo//f39//v7+//7+/v/+/v7//Rwof/7+/v/+/v7//v7+//7+/v/+/v + 7//Fr1v/rYwG/62MBv+8oz3/7+/v/+/v7//v7+//7+/v/52d1//v7+//7+/v/+7u7v/WyZb/7+/v/7S0 + 3v+JidL/4eHr/+/v7//+/v7/5eXl//n5+f/o6Oj/9/f3/+/v7//v7+//7+/v/9HCh//v7+//7+/v/+/v + 7//v7+//7+/v/869e/+tjAb/rYwG/5J/Xf/Pz+b/7+/v/+/v7//v7+//mJjV/+/v7//v7+//7u7u/7Kl + jP+Cgs//v7/h/+/v7//v7+//7+/v//7+/v/l5eX/+vr6/+jo6P/39/f/7+/v/+/v7//v7+//0cKH/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/9THkv/SxIr/7u7u/56e2P+Bgc//5ubs/+/v7/+SktT/7+/v/+Tk + 7P+Cgs//s6aM/+/v7//v7+//7+/v/+/v7//v7+///v7+/+bm5v/6+vr/6Ojo//f39//v7+//7+/v/+/v + 7//Rwof/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+zs7v91dcz/lpbV/4yM + 0/+SktT/eXnN/+vr7f/WyZb/7+/v/+/v7//v7+//7+/v/+/v7//+/v7/5ubm//r6+v/n5+f/9/f3/+/v + 7//v7+7/0MKG/7OVHP++pkT/39i7/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+7u + 7/+IiNH/Dg6v/5qa1v+5ud//RES+/yIhrv+FhdH/7u7u/+/v7//v7+//7+/v//7+/v/l5eX/+vr6/+fn + 5//39/f/7+/v/9PGj/+tjAb/rYwG/62MBv+ujQr/6ujh/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//Q0Ob/d3fM/7S03v+NjdP/oqLZ/wgIrf8GBq3/Bgat/wYGrf+Pj9P/7+/v/+/v7//v7+///v7+/+Xl + 5f/6+vr/5ubm//f39//v7+//wqxS/62MBv+tjAb/rYwG/62MBv/XzJ3/7+/v/+/v7//v7+//7+/v/+/v + 7//q6u3/iIjR/5eX1f/v7+//7+/v/5KS1P+cnNf/Bgat/wYGrf8GBq3/Bgat/1JSwv/v7+//7+/v/+/v + 7//+/v7/5eXl//r6+v/m5ub/9/f3/+/v7//KuG//rYwG/62MBv+tjAb/rYwG/9DChv/q593/7+/v/+/v + 7//v7+//oaHY/4iI0f/q6u3/7+/v/+/v7//v7+//kpDC/7qwo/8GBq3/Bgat/wYGrf8GBq3/Y2PG/+/v + 7//v7+//7+/v//7+/v/l5eX/+vr6/+fn5//39/f/7+/v/+ro3/+5njL/rYwG/62MBv/Mu3X/7+/v/9rR + qP/ZzaH/v7/g/4uL0v/Py83/1cmX/+fk1f/v7+//7u7t/9nNov+poJ7/7+/v/3l5zf8GBq3/Bgat/zIy + uf+3t9//n5/Y/+/v7//v7+///v7+/+Xl5f/6+vr/5+fn//f39//v7+//7+/v/+/v7//n49T/6+nj/+/v + 7//v7+//39/q/5ST0v/Nwpv/zLt1/62MBv+tjAb/rYwG/8y8eP/XzJ7/7u7t/5+f2P/v7+//7+/v/+Pj + 6//U1Of/7+/v/+/v7//v7+//7+/v/+/v7//+/v7/5eXl//r6+v/o6Oj/9/f3/+/v7//v7+//7+/v/+/v + 7//v7+//7u7u/6am2v+oqNr/7+/v/+/v7/+vkBD/rYwG/62MBv+tjAb/sJAR/+/v7//v7+//pKTZ/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v/m5ub/+vr6/+fn5//39/f/7+/v/+/v + 7//v7+//7+/v/7W13v+mptr/7Ozu/+/v7//v7+//7+/v/62MBv+tjAb/rYwG/62MBv+tjAb/7u7u/+/v + 7/+pqdv/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+/+bm5v/6+vr/6urq//f3 + 9//v7+//7+/v/+/v7//BweH/39/q/+/v7//v7+//7+/v/+/v7//v7+//wqtR/62MBv+tjAb/rYwG/8Ww + XP/v7+//7+/v/66u2//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7/5ubm//r6 + +v/q6ur/9/f3/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//u7u3/zr17/8Ou + Vv/Ov37/7u7t/+/v7//v7+//srLd/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+ + /v/m5ub/+vr6/+vr6//39/f/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+4uN//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+///v7+/+bm5v/6+vr/7e3t//j4+P/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/+/v7/5ubm//r6+v/t7e3///////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////m5ub//v7+//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4 + +P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4 + +P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//v7+/8AAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEA + CAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAACtjAb/ro0I/66OCv+vjwz/r5AO/7CQEP+xkRL/spQZ/7OU + Gv+zlRz/tZgi/7mfMv+6nzP/vKI6/7yjPP+9pD7/koBe/7+nRf/Bqkz/wqxR/8OsUv/Erlf/xbBc/8u5 + b//LunL/zLtz/8y7dv/Nu3b/zbx5/869ev/Ovnz/z79//wYGrf8ICK7/Dg6v/yMhrv8yMrr/RUW//1JS + w/9jY8f/dnbN/3h4zf95ec3/sqWN/7Snjf+qoJ//raWp/7qxo/+9s6P/0cGE/9HChv/Sw4f/0sSK/9PE + i//Txo7/1MaP/87DnP/VyJP/1smX/9bKl//Wypj/2Myd/9nOov/b0an/29Kq/97Vsv/g2Lr/k5DD/4KC + 0P+Dg9D/hobR/4iI0f+JidL/iorS/4uL0/+MjNP/jo7U/4+P1P+UlNP/kpLU/5OT1f+Xl9b/mJjW/5mZ + 1v+bm9f/nJzX/52d1/+fn9n/oKDY/6Gh2f+iotn/o6PZ/6Wl2v+mptr/p6fa/6io2/+pqdv/rKzc/66u + 3P+2tdT/sLDd/7Oz3v+0tN7/tbXe/7a23/+3t9//uLjg/7q64P+/v+H/y8bG/9DLzv/R0dH/0tLS/9jY + 2P/n4tH/6OTW/+nm2f/q6N7/6+jf/8DA4v/BweL/wsLi/8XF4//IyOT/zMzl/9DQ5v/U1Of/39/r/+Xl + 5f/m5ub/5+fn/+vp4f/r6uL/7Orj/+zq5f/g4Ov/4uLr/+Li7f/k5Oz/5eXs/+bm7P/o6Oj/6enp/+rq + 6v/r6+v/7ezo/+7t6v/r6+7/7Ozs/+3t7f/u7ez/7u7s/+/u7v/x8fH/8vLy//f39//4+Pj/+fn5//r6 + +v/7+/v//Pz8//39/f//////AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/oXFwcHBwcHBwcHBwcG9vb3BwcHBwcHBwcHBwcG9v + b5SekJ+foaGfoKCgnp+fn6CgnqChoKCfn6Ggn5+fn5+egZ6Vm5qZmpmZmZqampmampmZmpqaiZmampqa + mZqZmaKCnpWbmJmVmZmZlZmVmXQSBBJzmJd8mZWYmJWZmJmZooGej5uZlZmVlZWZlZmYDwAAAA6YmXuV + mZWZmZiYmJiigZ6Pm5WZlZmZmZWZmJcAAAAAAZeYepmVmZiYmZWZlaKBno2bmZWZlZiVmZiUPgoAAAAK + PZR3mJmYmJiVmZiZooCejZuYmZiZlZmYQD6VQgkAB0KZPjCYmJiZmJmVmJWigJ6Nm5iYHAYLNDyGmZiZ + lZGZlZWZYzw2DAUemJmZmaKAno2bmTQAAAABdZmVmJWZmZWZmZVmgwMAAAA0lZiVooCegZuVEwAAAAA9 + lZmYmZWVmZWVmWQ6AAAAABOZl5migJ6Cm5kYAAAAAG2ZmJmYmZmVmZmVYXIAAQAAGZWZlaKBnoKbmJEO + AAIxaFSMlZWYmJmVlZlemTMBAA2GmZiZooKejZuYmJkxlZmZlVYuBQJBlZmZlVqVlZc5lZiYWXiigZ6O + m5mYlTSVmZWZlRYAAA+ZlZWZVpmZlTqZZkeHlaKBno6bmJiZM5iYmZiZHgAAEH2ZmZVTlZiZK0V4mZmZ + ooCejpuVmZgzmJmYlZWZOTWVV0SMmVCZi0Qsl5WVlZWigZ6Om5mYmDOYmJiZmZWZlZmVkyhRS1AplDyZ + mZmZmaKBno2blZgyCRFCmJWVmZWZlZmZmEciVGslI0WYlZWVooCegpuZNwAAAAODmZmVmZWZl30qaExb + ISAgIE2ZmZmigJ6Cm5UUAAAAAD2YlZmVmZNHUpWVT1UgICAgJpeVlaKAnoKbmRcAAAAAMnWZl5laR5OV + mZlDLyAgICAnmZmZooGegpuVdQsBARqZQD53Sm48c5mVPi2VKSAgJGdWlZWigZ6Nm5mZmXKFlZh/Tjga + AAABHD2YWJmZin6ZmZmZmaKBno2blZWVmZiZX12VlQUAAAAGmZVclZWVlZWVlZWVooGejZuZmZmYZ12V + mZmZAQAAAACYmWCZmZmZmZmZmZmigp6Pm5WVlXd/lZWVlZUTAAAAFpWVYpWVlZWVlZWVlaKCno+bmZmZ + mZmZmZmZmZceFR+YmZllmZmZmZmZmZmZooKelJuYmJiYmJiYmJiYmJiYmJiYmGqYmJiYmJiYmJiigp6V + nZqampqampqampqampqampqampqampqampqamqKBn5WioqKioqKioqKioqKioqKioqKioqKioqKioqKi + ooKinZycnJycnJycnJycnJycnJycnJycnJycnJycnJycoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAABAAAAAgAAAAAQAgAAAA + AAAACAAAAAAAAAAAAAAAAAAAAAAAAOPj4//T09P/1dXV/9TU1P/U1NT/1NTU/9TU1P/U1NT/1NTU/9XV + 1f/U1NT/1NTU/9TU1P/U1NT/09PT/9DQ0P/o6Oj/7+/v/+/v7//v7+//7+/v/+/v7//v7+//08aO/+7t + 6//v7+//4eHr/+/v7//v7+//7+/v/+/v7//s7Oz/5eXl/+/v7//v7+//7+/v/+/v7//v7+//0MGE/62M + Bv+tjAb/7+/v/97e6v/v7+//7+/v/+/v7//v7+//6+vr/+Pj4//v7+//7+/v/+/v7//v7+//5N7I/+DZ + vv+tjAb/r44M/9XJlv/b2+j/7+/v/+/v7//v7+//7+/v/+rq6v/i4uL/7+/v/8WxX/+tjAb/y7lx/+/v + 7//v7+//7+/v/+/v7//v7+//2Njn/66NCv+tjAb/7+/v/+/v7//r6+v/4eHh/+/v7/+tjAb/rYwG/7me + Mf/v7+//7+/v/+/v7//v7+//7+/v/9fX5/+tjAb/rYwG/+bhz//v7+//6+vr/+Hh4f/v7+//7Ork/76l + Qv/v7+7/kZHU/+Havv/e1rT/7+/v/+/v7//S0uf/2M2h/86+ff/v7+//xcXj/+zs7P/j4+P/7+/v/+/v + 7//Rwof/7+/v/+/v7/+tjAb/rYwG/+/v7//v7+//z8/m/+/v7//t7e7/np7Y/+/v7//r6+v/5OTk/+/v + 7//v7+//0cKH/+/v7//v7+//7+/v/+/v7//FxeP/wMDh/8zM5f93d83/7+/v/+/v7//v7+//7Ozs/+Li + 4v/v7+//ybZp/62MBv/f17f/7+/v/+/v7//v7+//7+/v/4GBz/9tbcr/DQ2u/wYGrf/v7+//7+/v/+rq + 6v/h4eH/7+/v/62MBv+tjAb/tpkl/+/v7//v7+//yMjj/8XF4//v7+//0NDm/wYGrf8GBq3/zMzk/+/v + 7//r6+v/4uLi/+/v7//q6OD/xbBd/+7u7v/l4M7/lJG9/7qfNP/b0ar/1ciV/9bW5/+amtf/cnLL/+np + 7f/v7+//6+vr/+Pj4//v7+//7+/v/+/v7/+amtf/7+/v/869ev+tjAb/rYwG/+/v7//b2+n/7+/v/+/v + 7//v7+//7+/v/+zs7P/l5eX/7+/v/+/v7//u7u7/7+/v/+/v7//u7ez/rYwG/8CpS//v7+//4eHr/+/v + 7//v7+//7+/v/+/v7//s7Oz/6Ojo/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+vr + 7v/v7+//7+/v/+/v7//v7+//7Ozs//Hx8f/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4 + +P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4/+7u7v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAABAAAAAgAAAAAQAIAAAAAAAAAgAAAAAAAAAA + AAAAAAAAAAAAAK2MBv+ujQn/ro4L/7GSE/+xkhT/s5Ub/6qPKv+2mib/up80/7qgNf+7oTj/vaQ//7+q + XP/Bqkz/wKtc/8ayYP/LuXD/zLp0/8+/ff8GBq3/LS22/0NDvv92ds3/f3/P/7mysf++uLj/0MGD/9LE + if/Ux5H/1ceQ/9fLmv/Pxab/2tCm/9vRqf/g2bv/4dm8/+Lbv/+fnMT/lJTV/6Wl2v+oqNv/qanb/7Oz + 3v+4uN//u7vg/76+4f+/v+L/ysTC/8bCyP/Lx8v/2tPA/8rI1f/Z1dD/19XZ/+Pcwv/j3sb/5N7H/+Tf + yf/m4tH/6OTW/+jl2P/BweL/w8Pi/8XF4//KyuT/y8vl/83N5f/Q0Ob/19fo/9vb6f/d3er/5ubm/+fn + 5//g4Ov/4eHr/+Pj7f/k5O3/5+ft/+jo6P/u7ev/7+7r/+zs7v/u7u3/7+/u//Dw8P/x8fH/8vLy//Pz + 8//09PT/9vb2//j4+P/8/Pz/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP9SR0hHSEdHR0dIR0dHR0dOWFhUVFRUTxw2VExUVFRUVlZXVVNUUxsABFNFVFRU + VFZVV1FUUjkeAgs5NVJSU1JWVVcSBBpSVFJSUTEIBztUVlVXCAAMVFNUVFQwAQAhVFZVVyQPND0fIFNT + QB4QT0JWVVdUIlRUCgZNVD9SGSxRVlVXUyJSUjw7LisoKjJUVFZVVxoFIVRUU00oFhUURFNWVFcIAA1P + Uz0tUyUTEydUVlVXIhA6LxgOHDczJhdBUlZVV1RKQEYSAAFUQVRUVFRWVldTSVNTOAoRU0NTU1NTV1dY + VVVVVVVVVVVLVVVVVVZaW1tbW1tbW1tbW1tbW1tZAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== + + + \ No newline at end of file From 3327a31535ccafa47822af6112bcf001d7ba1824 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 19 Nov 2025 14:38:50 +0000 Subject: [PATCH 136/142] Task/rdmp 328 extraction archive triggers (#2254) * add extraction db triggers * fix test * add tests * add missing test * update tests * move checks --- CHANGELOG.md | 1 + ...atabaseMSSqlDestinationReExtractionTest.cs | 10 +- ...eMSSqlDestinationWithArchiveTriggerTest.cs | 777 ++++++++++++++++++ .../DataTableUploadDestinationTests.cs | 2 +- .../ExecuteFullExtractionToDatabaseMSSql.cs | 64 +- .../DataTableUploadDestination.cs | 107 ++- 6 files changed, 889 insertions(+), 72 deletions(-) create mode 100644 Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 19a9e6f65f..86a2bc6ced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Require all deletes to enter a commit message when using the commit system - Introduce ability to view Catalogues in a flat view - Fix bug where Internal catalogues were still able to be added to a CIC +- Add ability to use archive trigger when re-releasing to a database ## [9.0.3] - 2025-11-03 - Improve checking for default pipelines diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs index f1d4a30ac1..bbdf06a654 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs @@ -533,7 +533,7 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs() dt = destinationTable.GetDataTable(); - Assert.That(dt.Rows, Has.Count.EqualTo(2)); + Assert.That(dt.Rows, Has.Count.EqualTo(3)); } [Test] @@ -802,7 +802,7 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK() dt = destinationTable.GetDataTable(); - Assert.That(dt.Rows, Has.Count.EqualTo(2)); + Assert.That(dt.Rows, Has.Count.EqualTo(3)); } [Test] @@ -1061,7 +1061,7 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK() dt = destinationTable.GetDataTable(); - Assert.That(dt.Rows, Has.Count.EqualTo(2)); + Assert.That(dt.Rows, Has.Count.EqualTo(3)); } [Test] @@ -1209,6 +1209,8 @@ public void ExtractToDatabaseUseTriggers() var argumentDbNamePattern = destinationArguments.Single(static a => a.Name == "DatabaseNamingPattern"); var argumentTblNamePattern = destinationArguments.Single(static a => a.Name == "TableNamingPattern"); var reExtract = destinationArguments.Single(static a => a.Name == "AppendDataIfTableExists"); + var argUseArchiveTrigger = destinationArguments.Single(static a => a.Name == "UseArchiveTrigger"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); var extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) { @@ -1224,6 +1226,8 @@ public void ExtractToDatabaseUseTriggers() argumentDbNamePattern.SaveToDatabase(); argumentTblNamePattern.SetValue("$c_$d"); argumentTblNamePattern.SaveToDatabase(); + argUseArchiveTrigger.SetValue(true); + argUseArchiveTrigger.SaveToDatabase(); reExtract.SetValue(true); reExtract.SaveToDatabase(); diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs new file mode 100644 index 0000000000..34d50b3d06 --- /dev/null +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs @@ -0,0 +1,777 @@ +// Copyright (c) The University of Dundee 2018-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using FAnsi; +using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; +using Microsoft.Data.SqlClient; +using NUnit.Framework; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands.CatalogueCreationCommands; +using Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands; +using Rdmp.Core.CommandLine.DatabaseCreation; +using Rdmp.Core.CommandLine.Options; +using Rdmp.Core.CommandLine.Runners; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Aggregation; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.Curation.Data.Pipelines; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; +using Rdmp.Core.DataExport.DataExtraction.Pipeline.Sources; +using Rdmp.Core.DataFlowPipeline; +using Rdmp.Core.DataLoad.Triggers; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; +using SynthEHR; +using SynthEHR.Datasets; +using System; +using System.Data; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using Tests.Common; +using Tests.Common.Scenarios; +using TypeGuesser; + +namespace Rdmp.Core.Tests.DataExport.DataExtraction; + +public class ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest : DatabaseTests +{ + + private FileInfo CreateFileInForLoading(string filename, int rows, Random r) + { + var fi = new FileInfo(Path.Combine(Path.GetTempPath(), Path.GetFileName(filename))); + + var demog = new Demography(r); + var people = new PersonCollection(); + people.GeneratePeople(500, r); + + demog.GenerateTestDataFile(people, fi, rows); + + return fi; + } + + [Test] + public void SQLServerDestinationWithTriggersNoUpdate() { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); + + ec.SaveToDatabase(); + + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger= destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + var hicLoadID = dt.Rows[0].ItemArray[38]; + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[38], Is.EqualTo(hicLoadID)); + + } + + [Test] + public void SQLServerDestinationWithTriggersAndUpdate() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); + + ec.SaveToDatabase(); + + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 2"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + var hicLoadID = dt.Rows[0].ItemArray[38]; + + //update source here + var server = catalogue.GetDistinctLiveDatabaseServer(ReusableLibraryCode.DataAccess.DataAccessContext.InternalDataProcessing, false, out var dap); + if (server.Exists()) + { + var fdb = server.ExpectDatabase(catalogue.CatalogueItems.First().ColumnInfo.TableInfo.Database); + var ftbl = fdb.ExpectTable("bob"); + using (var con = server.GetConnection()) + { + con.Open(); + var sql = $"UPDATE {ftbl.GetFullyQualifiedName()} set notes = 'JOHN'"; + var cmd1 = server.GetCommand(sql, con); + cmd1.ExecuteNonQuery(); + } + + } + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[38], Is.Not.EqualTo(hicLoadID)); + } + [Test] + public void SQLServerDestinationWithTriggersNoPKs() { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = false; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); + + ec.SaveToDatabase(); + + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 1"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + var hicLoadID = dt.Rows[0].ItemArray[38]; + + //update source here + var server = catalogue.GetDistinctLiveDatabaseServer(ReusableLibraryCode.DataAccess.DataAccessContext.InternalDataProcessing, false, out var dap); + if (server.Exists()) + { + var fdb = server.ExpectDatabase(catalogue.CatalogueItems.First().ColumnInfo.TableInfo.Database); + var ftbl = fdb.ExpectTable("bob"); + using (var con = server.GetConnection()) + { + con.Open(); + var sql = $"UPDATE {ftbl.GetFullyQualifiedName()} set notes = 'JOHN'"; + var cmd1 = server.GetCommand(sql, con); + cmd1.ExecuteNonQuery(); + } + + } + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(2)); + } + +} \ No newline at end of file diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs index ffe63b1da1..95382b5950 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs @@ -1844,7 +1844,7 @@ public void DataTableUploadClashUpdateDropColumnWithTrigger() using var resultDt = table.GetDataTable(); Assert.That(resultDt.Rows, Has.Count.EqualTo(1)); Assert.That(resultDt.Rows[0].ItemArray[0], Is.EqualTo("Fish")); - Assert.That(resultDt.Rows[0].ItemArray[1], Is.EqualTo(string.Empty)); + Assert.That(resultDt.Rows[0].ItemArray[1].ToString(), Is.EqualTo(String.Empty)); table = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests_Archive"); using var resultDt2 = table.GetDataTable(); Assert.That(resultDt2.Rows, Has.Count.EqualTo(1)); diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs index 714b6edf44..3b67f5332f 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs @@ -4,12 +4,6 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text.RegularExpressions; using FAnsi.Discovery; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; @@ -18,7 +12,10 @@ using Rdmp.Core.DataExport.DataRelease.Pipeline; using Rdmp.Core.DataExport.DataRelease.Potential; using Rdmp.Core.DataFlowPipeline; +using Rdmp.Core.DataLoad.Engine.Job.Scheduling; using Rdmp.Core.DataLoad.Engine.Pipeline.Destinations; +using Rdmp.Core.DataLoad.Triggers.Exceptions; +using Rdmp.Core.DataLoad.Triggers.Implementations; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; @@ -26,6 +23,12 @@ using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.DataAccess; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; using YamlDotNet.Core; namespace Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; @@ -109,6 +112,10 @@ You must have either $a or $d [DemandsInitialization("An optional list of columns to index on e.g \"Column1, Column2\"")] public string UserDefinedIndex { get; set; } + + [DemandsInitialization("When writing to an existing database, will update records and store historical versions via a view", DefaultValue = false)] + public bool UseArchiveTrigger { get; set; } + private DiscoveredDatabase _destinationDatabase; private DataTableUploadDestination _destination; @@ -177,6 +184,7 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis var existing = _destinationDatabase.ExpectTable(tblName); if (existing.Exists()) { + var hasPKs = existing.DiscoverColumns().Any(col => col.IsPrimaryKey); if (_request.IsBatchResume) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, @@ -194,6 +202,26 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis // since we dropped it we should treat it as if it was never there to begin with _tableDidNotExistAtStartOfLoad = true; } + else if (UseArchiveTrigger && hasPKs) + { + + TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer); + var implementor = triggerFactory.Create(existing); + bool present; + try + { + present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + } + catch (TriggerMissingException) + { + present = false; + } + if (!present) + { + implementor.CreateTrigger(ThrowImmediatelyCheckNotifier.Quiet); + } + + } else { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, @@ -223,11 +251,13 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis _destination.IncludeTimeStamp = IncludeTimeStamp; _destination.UseTrigger = AppendDataIfTableExists; _destination.IndexTables = IndexTables; + _destination.UseTrigger = UseArchiveTrigger; _destination.IndexTableName = GetIndexName(); if (UserDefinedIndex is not null) _destination.UserDefinedIndexes = UserDefinedIndex.Split(',').Select(i => i.Trim()).ToList(); _destination.PreInitialize(_destinationDatabase, listener); + return _destination; } @@ -343,7 +373,7 @@ private string GetIndexName() } - return indexName.Replace(" ",""); + return indexName.Replace(" ", ""); } private string GetTableName(string suffix = null) @@ -664,6 +694,8 @@ public override void Check(ICheckNotifier notifier) $"Catalogue '{dsRequest.Catalogue}' does not have an Acronym but TableNamingPattern contains $a", CheckResult.Fail)); + + base.Check(notifier); try @@ -671,6 +703,24 @@ public override void Check(ICheckNotifier notifier) var server = DataAccessPortal.ExpectServer(TargetDatabaseServer, DataAccessContext.DataExport, false); var database = _destinationDatabase = server.ExpectDatabase(GetDatabaseName()); + if (UseArchiveTrigger) + { + if (_request is ExtractDatasetCommand dsRequest) + { + var existing = _destinationDatabase.ExpectTable(dsRequest.Catalogue.Name); + if (existing.Exists()) + { + var hasPKs = existing.DiscoverColumns().Any(col => col.IsPrimaryKey); + if (!hasPKs) + { + notifier.OnCheckPerformed(new CheckEventArgs( + $"Catalogue does not have any PKS. Cannot apply the archive trigger", + CheckResult.Fail)); + } + } + } + } + if (database.Exists()) { notifier.OnCheckPerformed( diff --git a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs index 913650fee7..9ecaca4fd1 100644 --- a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs +++ b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs @@ -4,22 +4,17 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Data; -using System.Diagnostics; -using System.Globalization; -using System.Linq; -using System.Threading.Tasks; +using FAnsi; using FAnsi.Connections; using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; using FAnsi.Discovery.TableCreation; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; -using Rdmp.Core.DataLoad.Triggers.Implementations; using Rdmp.Core.DataLoad.Triggers; +using Rdmp.Core.DataLoad.Triggers.Implementations; using Rdmp.Core.Logging; using Rdmp.Core.Logging.Listeners; using Rdmp.Core.Repositories.Construction; @@ -27,8 +22,14 @@ using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.DataAccess; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; using TypeGuesser; -using FAnsi; namespace Rdmp.Core.DataLoad.Engine.Pipeline.Destinations; @@ -145,38 +146,6 @@ private static object[] FilterOutItemAtIndex(object[] itemArray, int[] indexes) if (indexes.Length == 0) return itemArray; return itemArray.Where((source, idx) => !indexes.Contains(idx)).ToArray(); } - private string GetPKValue(DataColumn pkColumn, DataRow row) - { - var pkName = pkColumn.ColumnName; - var value = row[pkName]; - if (_externalCohortTable is not null) - { - var privateIdentifierField = _externalCohortTable.PrivateIdentifierField.Split('.').Last()[1..^1];//remove the "[]" from the identifier field - var releaseIdentifierField = _externalCohortTable.ReleaseIdentifierField.Split('.').Last()[1..^1];//remove the "[]" from the identifier field - if (pkName == releaseIdentifierField) - { - //going to have to look up the previous relaseID to match - DiscoveredTable cohortTable = _externalCohortTable.DiscoverCohortTable(); - using var lookupDT = cohortTable.GetDataTable(); - var releaseIdIndex = lookupDT.Columns.IndexOf(releaseIdentifierField); - var privateIdIndex = lookupDT.Columns.IndexOf(privateIdentifierField); - var foundRow = lookupDT.Rows.Cast().Where(r => r.ItemArray[releaseIdIndex].ToString() == value.ToString()).LastOrDefault(); - if (foundRow is not null) - { - var originalValue = foundRow.ItemArray[privateIdIndex]; - var existingIDsforReleaseID = lookupDT.Rows.Cast().Where(r => r.ItemArray[privateIdIndex].ToString() == originalValue.ToString()).Select(s => s.ItemArray[releaseIdIndex].ToString()); - if (existingIDsforReleaseID.Count() > 0) - { - //we don't know what the current release ID is ( there may be ones from multiple cohorts) - var ids = existingIDsforReleaseID.Select(id => $"'{id}'"); - return $"{pkName} in ({string.Join(',', ids)})"; - } - } - } - } - return $"{pkName} = '{value}'"; - } - public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) @@ -393,29 +362,45 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener } - - foreach (DataRow row in rowsToModify.Distinct()) + if (rowsToModify.Any()) { - //replace existing - var args = new DatabaseOperationArgs(); - List columns = []; - foreach (DataColumn column in toProcess.Columns) + var tmpDt = toProcess.Clone(); + foreach (var row in rowsToModify) + { + tmpDt.ImportRow(row); + } + var tblName = $"tmpTable_{DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond}"; + var tmpTable = _discoveredTable.Database.CreateTable(tblName, tmpDt); + + var customLines = new List() { }; + foreach (var col in tmpDt.Columns.Cast()) + { + if (!pkColumns.Select(c => c.ColumnName).Contains(col.ColumnName)) + { + customLines.Add(new CustomLine($"t1.{col.ColumnName} = t2.{col.ColumnName}", QueryComponent.SET)); + } + else + { + customLines.Add(new CustomLine($"t1.{col.ColumnName} = t2.{col.ColumnName}", QueryComponent.JoinInfoJoin)); + customLines.Add(new CustomLine($"t1.{col.ColumnName} = t2.{col.ColumnName}", QueryComponent.WHERE)); + } + } + var existingColumns = _discoveredTable.DiscoverColumns().Select(c => c.GetRuntimeName()).Where(c => c != SpecialFieldNames.DataLoadRunID && c != SpecialFieldNames.ValidFrom); + var columnsThatPreviouslyExisted = existingColumns.Where(ec => !toProcess.Columns.Cast().ToList().Select(c => c.ColumnName).Contains(ec)); + foreach (var col in columnsThatPreviouslyExisted) + { + customLines.Add(new CustomLine($"t1.{col} = t2.{col}", QueryComponent.SET)); + + } + + var str = _discoveredTable.Database.Server.GetQuerySyntaxHelper().UpdateHelper.BuildUpdate(_discoveredTable, tmpTable, customLines); + using (var connection = _discoveredTable.Database.Server.GetConnection()) { - //if (!pkColumns.Contains(column)) - //{ - columns.Add(column.ColumnName); - //} + connection.Open(); + var cmd = _discoveredTable.Database.Server.GetCommand(str, connection); + cmd.ExecuteNonQuery(); + tmpTable.Drop(); } - //need to check for removed column and null them out - var existingColumns = _discoveredTable.DiscoverColumns().Select(c => c.GetRuntimeName()); - var columnsThatPreviouslyExisted = existingColumns.Where(c => !pkColumns.Select(pk => pk.ColumnName).Contains(c) && !columns.Contains(c) && c != SpecialFieldNames.DataLoadRunID && c != SpecialFieldNames.ValidFrom); - var nullEntries = string.Join(" ,", columnsThatPreviouslyExisted.Select(c => $"{c} = NULL")); - var nullText = nullEntries.Length > 0 ? $" , {nullEntries}" : ""; - var columnString = string.Join(" , ", columns.Select(col => $"{col} = '{row[col]}'").ToList()); - var pkMatch = string.Join(" AND ", pkColumns.Select(pk => GetPKValue(pk, row)).ToList()); - var sql = $"update {_discoveredTable.GetFullyQualifiedName()} set {columnString} {nullText} where {pkMatch}"; - var cmd = _discoveredTable.GetCommand(sql, args.GetManagedConnection(_discoveredTable).Connection); - cmd.ExecuteNonQuery(); } foreach (DataRow row in rowsToModify.Distinct()) From bd1abfeb4f3f24025d6a67f3022cf2e7124ef4e9 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 24 Nov 2025 12:09:39 +0000 Subject: [PATCH 137/142] Prep v9.1.0 (#2272) * bump packages * update sign tool * use old azure sign version * update version --- .github/workflows/build.yml | 2 +- CHANGELOG.md | 2 +- Directory.Packages.props | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39c5aae7e8..b2f7e3876d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -204,7 +204,7 @@ jobs: - name: Sign & zip shell: bash run: | - dotnet tool install --global AzureSignTool + dotnet tool install --global --version 6.0.1 AzureSignTool AzureSignTool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v PublishWindows/rdmp.exe AzureSignTool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v PublishWinForms/ResearchDataManagementPlatform.exe mkdir -p dist diff --git a/CHANGELOG.md b/CHANGELOG.md index 86a2bc6ced..87e589535b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [9.1.0] - Unreleased +## [9.1.0] - 2025-11-24 - Fix bug with duplicate searchables - Improve UI for committing cohorts across projects - Add the ability to Template Cohort Identification Configurations (see [Documentation\cohorts\CohortIdentificationConfigurationTemplates.md]) diff --git a/Directory.Packages.props b/Directory.Packages.props index 311d122b1d..9fe5afe69b 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,9 +1,9 @@ - - - - + + + + @@ -17,14 +17,14 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + - + - + @@ -43,7 +43,7 @@ - + \ No newline at end of file From 00a3e538c25dc9d6a68ae0c6939b1126fc680f19 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 25 Nov 2025 13:01:21 +0000 Subject: [PATCH 138/142] allow service workers --- .../ExecuteCommandExportCataloguesToConfluence.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs index 17f16fdae6..6de7b7ce27 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs @@ -22,6 +22,7 @@ public class ExecuteCommandExportCataloguesToConfluence : BasicCommandExecution, private readonly string _apiKey; private readonly string _owner; private readonly string _description; + private readonly bool _isServiceAccount; private readonly HttpClient _client = new(); private readonly Dictionary _cataloguePageLookups = []; private string rootParentId = null; @@ -91,7 +92,7 @@ private class ConfluencePageResponse public ConfluencePageVersion version { get; set; } } - public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, string subdomain, int spaceId, string apiKey, string owner, string description) + public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, string subdomain, int spaceId, string apiKey, string owner, string description, bool isServiceaccount) { _activator = activator; _subdomain = subdomain; @@ -99,6 +100,7 @@ public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, _apiKey = apiKey; _owner = owner; _description = description; + _isServiceAccount = isServiceaccount; } private static ConfluencePageResponse ResponseToConfluenceResponseObject(HttpResponseMessage response) @@ -162,6 +164,7 @@ private void CreateContainerPage(ConfluencePageBuilder builder, string uri) title = $"{_owner} Catalogues", body = new ConfluencePageBody() { value = rootPageHTML } }; + var x = request.ToString(); HttpResponseMessage response = Task.Run(async () => await _client.PostAsJsonAsync(uri, request)).Result; if (response.StatusCode == System.Net.HttpStatusCode.OK) { @@ -256,8 +259,7 @@ public override void Execute() .Where(c => !c.IsDeprecated && !c.IsInternalDataset && !c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) .ToList(); var builder = new ConfluencePageBuilder(catalogues, _owner, _description, _subdomain); - var uri = $"https://{_subdomain}.atlassian.net/wiki/api/v2/pages"; - + var uri = _isServiceAccount ? $"https://api.atlassian.com/ex/confluence/{_subdomain}/api/v2/pages" : $"https://{_subdomain}.atlassian.net/wiki/api/v2/pages"; CreateContainerPage(builder, uri); if (rootParentId != null) { From a4d245fb7568e11b9dc3ffe9fcf2100fd6d93e40 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 25 Nov 2025 15:18:18 +0000 Subject: [PATCH 139/142] update confluence builder --- ...cuteCommandExportCataloguesToConfluence.cs | 10 ++++++-- .../Confluence/ConfluencePageBuilder.cs | 24 ++++++++++++------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs index 6de7b7ce27..82ff2a6b77 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs @@ -23,6 +23,7 @@ public class ExecuteCommandExportCataloguesToConfluence : BasicCommandExecution, private readonly string _owner; private readonly string _description; private readonly bool _isServiceAccount; + private readonly int? _rootPageParent; private readonly HttpClient _client = new(); private readonly Dictionary _cataloguePageLookups = []; private string rootParentId = null; @@ -92,7 +93,7 @@ private class ConfluencePageResponse public ConfluencePageVersion version { get; set; } } - public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, string subdomain, int spaceId, string apiKey, string owner, string description, bool isServiceaccount) + public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, string subdomain, int spaceId, string apiKey, string owner, string description, bool isServiceaccount, int? rootPageParent = null) { _activator = activator; _subdomain = subdomain; @@ -101,6 +102,7 @@ public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, _owner = owner; _description = description; _isServiceAccount = isServiceaccount; + _rootPageParent = rootPageParent; } private static ConfluencePageResponse ResponseToConfluenceResponseObject(HttpResponseMessage response) @@ -164,6 +166,10 @@ private void CreateContainerPage(ConfluencePageBuilder builder, string uri) title = $"{_owner} Catalogues", body = new ConfluencePageBody() { value = rootPageHTML } }; + if (_rootPageParent != null) + { + request.parentId = _rootPageParent.ToString() ; + } var x = request.ToString(); HttpResponseMessage response = Task.Run(async () => await _client.PostAsJsonAsync(uri, request)).Result; if (response.StatusCode == System.Net.HttpStatusCode.OK) @@ -203,7 +209,7 @@ private void CreateCataloguePage(Catalogue catalogue, ConfluencePageBuilder buil var request = new ConfluencePagePostRequest() { spaceId = _spaceId.ToString(), - title = catalogue.Name, + title = $"{catalogue.Name}{(catalogue.Acronym != null ? $" ({catalogue.Acronym})" : "")}", parentId = rootParentId, body = new ConfluencePageBody() { value = cataloguePageHTML } }; diff --git a/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs b/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs index 2972fdba9c..46858a1817 100644 --- a/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs +++ b/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs @@ -19,20 +19,21 @@ public class ConfluencePageBuilder private readonly string _repositoryName = ""; private readonly string _repositoryDescription = ""; private readonly string _subdomain = ""; - public ConfluencePageBuilder(List catalogues, string name,string description, string subdomain) { + public ConfluencePageBuilder(List catalogues, string name, string description, string subdomain) + { _catalogues = catalogues; _repositoryName = name; _repositoryDescription = description; _subdomain = subdomain; } - private string BuildCatalogueOverviewHTML(Catalogue catalogue,string pageId) + private string BuildCatalogueOverviewHTML(Catalogue catalogue, string pageId) { Console.WriteLine($"Building overview for catalogue '{catalogue.Name}'"); - string name = pageId != null ?$"{catalogue.Name}" : catalogue.Name; + string name = pageId != null ? $"{catalogue.Name}" : catalogue.Name; return $""" -

{catalogue.Name}

+

{catalogue.Name}{(catalogue.Acronym != null ? $" ({catalogue.Acronym})" : "")}

+ +
@@ -44,6 +45,9 @@ Dataset Name Tags + Acronym +
@@ -53,21 +57,25 @@ Dataset Name {catalogue.Description} - {string.Join(", ",catalogue.Search_keywords)} + {string.Join(", ", catalogue.Search_keywords)} + + {catalogue.Acronym}
"""; } - public string BuildContainerPage(Dictionary cataloguePageLookup) { + public string BuildContainerPage(Dictionary cataloguePageLookup) + { Console.WriteLine($"Building container page. Found {_catalogues.Count} catalogues."); return $"""

{_repositoryName} Catalogues

{_repositoryDescription}

- {string.Join("",_catalogues.Select(c => { cataloguePageLookup.TryGetValue(c.ID, out var pageId); return BuildCatalogueOverviewHTML(c, pageId); }))} + {string.Join("", _catalogues.Select(c => { cataloguePageLookup.TryGetValue(c.ID, out var pageId); return BuildCatalogueOverviewHTML(c, pageId); }))} """; } @@ -230,7 +238,7 @@ Null Possible (Y/N) Has Lookups - {string.Join("",catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation is not null).Select(ci => BuildDataVariableRecord(ci)))} + {string.Join("", catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation is not null).Select(ci => BuildDataVariableRecord(ci)))} """; } From bcedcc17c167980923e33da36f0c093f21b3db1f Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 25 Nov 2025 15:21:49 +0000 Subject: [PATCH 140/142] add changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87e589535b..f43dcd3d3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.1.1] - Unreleased +- Allow Atlassian service workers to write to Confluence from RDMP + ## [9.1.0] - 2025-11-24 - Fix bug with duplicate searchables - Improve UI for committing cohorts across projects From 8822be4ff585a77af41ff3065af77fd1b7e70cdd Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 26 Nov 2025 10:10:19 +0000 Subject: [PATCH 141/142] Update ExecuteCommandExportCataloguesToConfluence.cs --- .../AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs index 82ff2a6b77..443961c50a 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs @@ -170,7 +170,6 @@ private void CreateContainerPage(ConfluencePageBuilder builder, string uri) { request.parentId = _rootPageParent.ToString() ; } - var x = request.ToString(); HttpResponseMessage response = Task.Run(async () => await _client.PostAsJsonAsync(uri, request)).Result; if (response.StatusCode == System.Net.HttpStatusCode.OK) { From 4fcaf9d1192d36179b9cfbc41aad8872a433b760 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 2 Dec 2025 12:40:33 +0000 Subject: [PATCH 142/142] prep release (#2281) --- CHANGELOG.md | 2 +- SharedAssemblyInfo.cs | 6 +++--- rdmp-client.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f43dcd3d3d..a7242d3fff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [9.1.1] - Unreleased +## [9.1.1] - 2025-12-02 - Allow Atlassian service workers to write to Confluence from RDMP ## [9.1.0] - 2025-11-24 diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index c271079044..2b089aac54 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("9.1.0")] -[assembly: AssemblyFileVersion("9.1.0")] -[assembly: AssemblyInformationalVersion("9.1.0")] +[assembly: AssemblyVersion("9.1.1")] +[assembly: AssemblyFileVersion("9.1.1")] +[assembly: AssemblyInformationalVersion("9.1.1")] diff --git a/rdmp-client.xml b/rdmp-client.xml index f81a73d09e..a2832499f4 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -1,6 +1,6 @@ - 9.1.0.0 + 9.1.1.0 https://github.com/HicServices/RDMP/releases/download/v9.1.0/rdmp-9.1.0-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true