diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c213a232b1..b2f7e3876d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,9 +17,9 @@ jobs: runs-on: windows-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: setup .NET - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 - name: Populate Databases.yaml shell: bash run: | @@ -84,9 +84,9 @@ jobs: runs-on: windows-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: setup .NET - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 - name: Populate Databases.yaml shell: bash run: | @@ -152,15 +152,15 @@ jobs: runs-on: windows-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: setup .NET - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 - name: Determine RDMP build version id: version shell: bash run: perl -ne "print \"rdmpversion=\$1\n\" if /AssemblyInformationalVersion\(\"([0-9a-z.-]+)\"\)/;" SharedAssemblyInfo.cs >> $GITHUB_OUTPUT - name: Setup .NET Core - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 with: dotnet-version: 7.0.x - name: BundleSource @@ -204,7 +204,7 @@ jobs: - name: Sign & zip shell: bash run: | - dotnet tool install --global AzureSignTool + dotnet tool install --global --version 6.0.1 AzureSignTool AzureSignTool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v PublishWindows/rdmp.exe AzureSignTool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v PublishWinForms/ResearchDataManagementPlatform.exe mkdir -p dist @@ -216,7 +216,7 @@ jobs: - name: Install Perl dependencies - uses: shogo82148/actions-setup-perl@v1.34.0 + uses: shogo82148/actions-setup-perl@v1.36.0 with: install-modules-with: cpanm install-modules: Archive::Zip Archive::Tar @@ -293,7 +293,7 @@ jobs: path: ${{ github.workspace }}/ key: ${{ github.sha }}-your-cache-key-bundled - name: setup .NET - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 - name: Determine RDMP build version id: version shell: bash @@ -307,4 +307,4 @@ jobs: file: dist/* tag: ${{ github.ref }} overwrite: true - file_glob: true + file_glob: true \ No newline at end of file diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 21f8bc3df3..cff7cefa04 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -24,15 +24,15 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: setup .NET - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 with: global-json-file: global.json dotnet-version: '6.0.x' - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} queries: +security-and-quality @@ -49,13 +49,13 @@ jobs: sed -i 's/"language": ""/"language": "en-US"/' sarif-results/scs.sarif - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{ matrix.language }}" upload: False output: sarif-results - name: Upload SARIF - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: sarif-results diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 33c0c80c72..b85738f672 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,11 +13,11 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: submodules: true - name: Setup .NET Core - uses: actions/setup-dotnet@v4.3.1 + uses: actions/setup-dotnet@v5.0.0 with: dotnet-version: 6.0.x - name: Cache Nuget diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index e149d9ac23..2f993feb21 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: submodules: recursive - name: Run link check diff --git a/Application/ResearchDataManagementPlatform/RDMPMainForm.cs b/Application/ResearchDataManagementPlatform/RDMPMainForm.cs index f181cbefa5..a018340aab 100644 --- a/Application/ResearchDataManagementPlatform/RDMPMainForm.cs +++ b/Application/ResearchDataManagementPlatform/RDMPMainForm.cs @@ -105,6 +105,7 @@ private void RDMPMainForm_Load(object sender, EventArgs e) _rdmpTopMenuStrip1.InjectButton(exceptionCounter); _windowManager = new WindowManager(_theme, this, _refreshBus, dockPanel1, RepositoryLocator, exceptionCounter); + SetItemActivator(_windowManager.ActivateItems); _rdmpTopMenuStrip1.SetWindowManager(_windowManager); diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs b/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs index 126afc435b..0c2250c1b1 100644 --- a/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs +++ b/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs @@ -55,6 +55,7 @@ using Rdmp.UI.Refreshing; using Rdmp.UI.Rules; using Rdmp.UI.SimpleDialogs; +using Rdmp.UI.SimpleDialogs.Cohorts; using Rdmp.UI.SimpleDialogs.ForwardEngineering; using Rdmp.UI.SingleControlForms; using Rdmp.UI.SubComponents; @@ -653,7 +654,6 @@ public override void Wait(string title, Task task, CancellationTokenSource cts) public override IEnumerable GetIgnoredCommands() { yield return typeof(ExecuteCommandRefreshObject); - yield return typeof(ExecuteCommandChangeExtractability); yield return typeof(ExecuteCommandOpenInExplorer); yield return typeof(ExecuteCommandDeletePlugin); yield return typeof(ExecuteCommandCreateNewFileBasedProcessTask); @@ -905,6 +905,12 @@ public override CohortCreationRequest GetCohortCreationRequest(ExternalCohortTab return ui.ShowDialog() == DialogResult.OK ? ui.Result : null; } + public override IProject CohortCommitProjectSelect(IProject currentProject,Project[] projects) + { + var ui = new CohortCommitProjectSelectionUI(this, currentProject, projects); + return ui.ShowDialog() == DialogResult.OK ? ui.Result : null; ; + } + public override CohortHoldoutLookupRequest GetCohortHoldoutLookupRequest(ExternalCohortTable externalCohortTable, IProject project, CohortIdentificationConfiguration cic) { diff --git a/CHANGELOG.md b/CHANGELOG.md index 1704a624f8..a7242d3fff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,40 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.1.1] - 2025-12-02 +- Allow Atlassian service workers to write to Confluence from RDMP + +## [9.1.0] - 2025-11-24 +- Fix bug with duplicate searchables +- Improve UI for committing cohorts across projects +- Add the ability to Template Cohort Identification Configurations (see [Documentation\cohorts\CohortIdentificationConfigurationTemplates.md]) +- Require all deletes to enter a commit message when using the commit system +- Introduce ability to view Catalogues in a flat view +- Fix bug where Internal catalogues were still able to be added to a CIC +- Add ability to use archive trigger when re-releasing to a database + +## [9.0.3] - 2025-11-03 +- Improve checking for default pipelines +- Improve confluence catalogue metadata extraction +- Improve project specific catalogue error handling + +## [9.0.2] - 2025-10-21 +- Add ability to view logs as a flat list. Default settings can be updated via user settings +- allow the use of Catalogue CIC filters in corresponding Extractions +- Add Data Load component to allow SQL to be executed on an external database server +- Improve Extraction Log Viewer Filter +- Add Dataset Variable document to extractions +- Update extractability to only allow non-internal projects to be extracted +- Remove functionality to mark individual catalogues as extractable/not extractable. Favor use of marking as internal. +- Move RSA key checks to fix bug where bad RSA keys were causing issues at launch +- Add cli command to export catalogue metadata to a Atlassian Confluence space +- Allow the use of network share RSA keys + + +## [9.0.1] - 2025-07-31 +- Improve Right-click performance +- Fix bug where orphaned filters were causing issues with cohort configurations + ## [9.0.0] - 2025-07-23 - Add ability to use Extraction Configuration ID in the naming scheme of the extract to database Pipeline component - Update Cohort Versioning Icons and Interface diff --git a/Directory.Packages.props b/Directory.Packages.props index 2eb0a64515..9fe5afe69b 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,31 +1,31 @@ - - - - + + + + - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - + + + + + + + + + @@ -33,17 +33,17 @@ - - + + - + - - + + - + \ No newline at end of file diff --git a/Documentation/CodeTutorials/CustomMetadataSubstitutions.md b/Documentation/CodeTutorials/CustomMetadataSubstitutions.md index 4a4834f594..a5a992fa9c 100644 --- a/Documentation/CodeTutorials/CustomMetadataSubstitutions.md +++ b/Documentation/CodeTutorials/CustomMetadataSubstitutions.md @@ -111,7 +111,6 @@ When running with `oneFile` off (or inside a `$foreach $Catalogue` block) the fo |$Geographical_coverage|User provided | |$Granularity|User provided | |$ID|RDMP allocated number uniquely identifying this object instance in this RDMP installation| -|$IsColdStorageDataset|True or False flag, configurable in RDMP| |$IsDeprecated|True or False flag, configurable in RDMP| |$IsInternalDataset|True or False flag, configurable in RDMP| |$Last_revision_date|User provided| diff --git a/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md b/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md new file mode 100644 index 0000000000..e5eea2410e --- /dev/null +++ b/Documentation/Cohorts/CohortIdentificationConfigurationTemplates.md @@ -0,0 +1,38 @@ +## Cohort Identification Configuration Templates + +Cohort Identification Configurations (CICs) can be turned into Templates for future re-use. +Templates offer certain functionality that differs from standard CICs. They: +* Cannot be edited +* Cannot be used to commit a cohort +* Are stored in a separate Templates section of the Cohort Builder + +Templates are designed to be a base to build your CIC from, saving you from having to copy, add, and edit boilerplate CIC configuration each and every time. + +### Creating a Template + +To create a new CIC Template: +1. Begin by creating a new CIC (or use an existing CIC). +2.Update your CIC to create the configuration you wish to template. +3. Once ready to turn this CIC into a template, right click on the CIC in the cohort builder tree. +4. Select the option "Create Cohort Identification Template". +5. If your CIC is associated with a [Project], it will ask you some questions about how you would like to associate the template +6. Congratulations, You've just made a CIC template + + +### Using a Template + +To use a CIC Template: +1. Right click on an existing CIC Template +2. Select the "Use Template Cohort Identification Configuration" option +3. If your template CIC is associated with a [Project], it will ask you some questions about how you would like to associate the new CIC +4. Congratulations, You've just made a new CIC from a template + + +### Associating with a [Project] + +Just like associating a CIC with a [Project], CIC Templates can also be associated with [Project]s. +While these templates can still be used outside the associated [Project], it does allow you to see the template from within the [Project] view. +This may be helpful to keep track of what templates are typically used with each [Project] + + +[Project]: ../CodeTutorials/Glossary.md#Project \ No newline at end of file diff --git a/Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs b/Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs new file mode 100644 index 0000000000..efee93e1b8 --- /dev/null +++ b/Rdmp.Core.Tests/CohortCreation/CohortIdentificationConfigurationTemplateTests.cs @@ -0,0 +1,97 @@ +using MathNet.Numerics; +using NUnit.Framework; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tests.Common; + +namespace Rdmp.Core.Tests.CohortCreation +{ + public class CohortIdentificationConfigurationTemplateTests: DatabaseTests + { + private static readonly Random random = new(); + private static string RandomString(int length) + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + return new string(Enumerable.Repeat(chars, length) + .Select(s => s[random.Next(s.Length)]).ToArray()); + } + + private CohortIdentificationConfiguration CreateBaseCIC(string name) + { + var cic = new CohortIdentificationConfiguration(CatalogueRepository, name); + cic.CreateRootContainerIfNotExists(); + cic.SaveToDatabase(); + return cic; + } + [Test] + public void CreateTemplateFromCIC_Test() + { + var cohortName = RandomString(10); + var cic = CreateBaseCIC(cohortName); + var cmd = new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(new ThrowImmediatelyActivator(RepositoryLocator), cic); + Assert.DoesNotThrow(()=>cmd.Execute()); + var foundTemplate = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName} Template"); + Assert.That(foundTemplate.Length,Is.EqualTo(1)); + } + + [Test] + public void CreateTemplateFromCICWithProject_Test() + { + var cohortName = RandomString(10); + var cic = CreateBaseCIC(cohortName); + var project = new Project(DataExportRepository, RandomString(10)); + project.SaveToDatabase(); + var cmd = new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(new ThrowImmediatelyActivator(RepositoryLocator), cic); + cmd.SetTarget(project); + Assert.DoesNotThrow(() => cmd.Execute()); + var foundTemplate = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName} Template"); + Assert.That(foundTemplate.Length, Is.EqualTo(1)); + var associations = RepositoryLocator.DataExportRepository.GetAllObjects().Where(a => a.CohortIdentificationConfiguration_ID == foundTemplate[0].ID).ToList(); + Assert.That(associations.Count, Is.EqualTo(1)); + Assert.That(associations[0].Project.Name, Is.EqualTo(project.Name)); + } + + [Test] + public void CreateCICFromTemplate_Test() + { + var cohortName = RandomString(10); + var cic = CreateBaseCIC(cohortName); + var ciccmd = new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(new ThrowImmediatelyActivator(RepositoryLocator), cic); + Assert.DoesNotThrow(() => ciccmd.Execute()); + var template = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName} Template").First(); + + var cmd = new ExecuteCommandUseTemplateCohortIdentificationConfiguration(new ThrowImmediatelyActivator(RepositoryLocator), template); + Assert.DoesNotThrow(() => cmd.Execute()); + var cics = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName}"); + Assert.That(cics.Length, Is.EqualTo(2)); + } + + [Test] + public void CreateCICFromTemplateWithProject_Test() + { + var cohortName = RandomString(10); + var cic = CreateBaseCIC(cohortName); + var ciccmd = new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(new ThrowImmediatelyActivator(RepositoryLocator), cic); + Assert.DoesNotThrow(() => ciccmd.Execute()); + var template = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName} Template").First(); + var project = new Project(DataExportRepository, RandomString(10)); + project.SaveToDatabase(); + var cmd = new ExecuteCommandUseTemplateCohortIdentificationConfiguration(new ThrowImmediatelyActivator(RepositoryLocator), template); + cmd.SetTarget(project); + Assert.DoesNotThrow(() => cmd.Execute()); + var cics = RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Name", $"{cohortName}"); + Assert.That(cics.Length, Is.EqualTo(2)); + var associations = RepositoryLocator.DataExportRepository.GetAllObjects().Where(a => cics.Select(c => c.ID).Contains(a.CohortIdentificationConfiguration_ID)).ToList(); + Assert.That(associations.Count, Is.EqualTo(1)); + Assert.That(associations[0].Project.Name, Is.EqualTo(project.Name)); + } + } +} diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs index 3a7169231c..f0c786c31a 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs @@ -26,7 +26,7 @@ public void TestDatasetCreationOKExtendedParameters() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset2","somedoi","some source"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset2" && ds.DigitalObjectIdentifier == "somedoi" && ds.Source == "some source"); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset2" && ds.DigitalObjectIdentifier == "somedoi" && ds.Source == "some source"); Assert.That(founddataset,Is.Not.Null); } } diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs index b3324ee241..52b79fc6d0 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs @@ -13,20 +13,20 @@ public void TestDeleteExistingDataset() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Has.Length.EqualTo(1)); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset"); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Has.Length.EqualTo(1)); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static ds => ds.Name == "dataset"); var delCmd = new ExecuteCommandDeleteDataset(GetMockActivator(), founddataset); Assert.DoesNotThrow(() => delCmd.Execute()); - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); } [Test] public void TestDeleteNonExistantDataset() { - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); - var founddataset = new Core.Curation.Data.Dataset(); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); + var founddataset = new Core.Curation.Data.Datasets.Dataset(); var delCmd = new ExecuteCommandDeleteDataset(GetMockActivator(), founddataset); Assert.Throws(() => delCmd.Execute()); - Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); + Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(), Is.Empty); } } \ No newline at end of file diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs new file mode 100644 index 0000000000..8d2889b1a2 --- /dev/null +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs @@ -0,0 +1,67 @@ +using NUnit.Framework; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.DataExport.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Tests.CommandExecution +{ + class ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests: CommandCliTests + { + [Test] + public void TestImportInternalCataloguesOnly() + { + var _cata1 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset1"); + _cata1.IsInternalDataset = true; + var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2"); + _cata1.SaveToDatabase(); + _cata2.SaveToDatabase(); + var ext = new ExtractableDataSet(GetMockActivator().RepositoryLocator.DataExportRepository, _cata2); + ext.SaveToDatabase(); + var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test",1,"test"); + var provider = new InternalDatasetProvider(GetMockActivator(),configuration,null); + var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false,true, false, false,false); + Assert.That(cmd.GetCatalogues().Count, Is.EqualTo(1)); + } + [Test] + public void TestImportProjectSpecificCataloguesOnly() + { + var _cata1 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset1"); + + var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2"); + _cata1.SaveToDatabase(); + _cata2.SaveToDatabase(); + + var ext = new ExtractableDataSet(GetMockActivator().RepositoryLocator.DataExportRepository, _cata1); + ext.SaveToDatabase(); + + var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test", 1, "test"); + var provider = new InternalDatasetProvider(GetMockActivator(), configuration, null); + var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false, false, true, false, false); + Assert.That(cmd.GetCatalogues().Count, Is.EqualTo(1)); + } + [Test] + public void TestImportDeprecatedCataloguesOnly() + { + var _cata1 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset1"); + _cata1.IsDeprecated = true; + var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2"); + var ext = new ExtractableDataSet(GetMockActivator().RepositoryLocator.DataExportRepository, _cata2); + ext.SaveToDatabase(); + + ext.SaveToDatabase(); + _cata1.SaveToDatabase(); + _cata2.SaveToDatabase(); + + var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test", 1, "test"); + var provider = new InternalDatasetProvider(GetMockActivator(), configuration, null); + var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false, false, false, true, false); + Assert.That(cmd.GetCatalogues().Count, Is.EqualTo(1)); + } + } +} diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs index 89b7dcb6b1..55ce585c61 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs @@ -34,15 +34,10 @@ public void TestLinkCatalogueToDataset() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(static c => c.Name == "Dataset1"); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), foundCatalogue, founddataset); Assert.DoesNotThrow(linkCmd.Execute); - var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(); - foreach (var ci in columInfo) - { - Assert.That(ci.Dataset_ID, Is.EqualTo(founddataset.ID)); - } founddataset.DeleteInDatabase(); foundCatalogue.DeleteInDatabase(); @@ -72,29 +67,17 @@ public void TestLinkCatalogueToDatasetNotAll() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(c => c.Name == "Dataset1"); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), foundCatalogue, founddataset, false); Assert.DoesNotThrow(linkCmd.Execute); - var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().Where(ci => _cata1.CatalogueItems.Contains(ci)); - foreach (var ci in columInfo) - { - Assert.That(ci.ColumnInfo.Dataset_ID, Is.EqualTo(founddataset.ID)); - } - - var columInfo2 = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().Where(ci => _cata2.CatalogueItems.Contains(ci)); - foreach (var ci in columInfo2) - { - Assert.That(ci.ColumnInfo.Dataset_ID, Is.Null); - } - } [Test] public void TestLinkCatalogueToDatasetBadCatalogue() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), null, founddataset, false); Assert.Throws(linkCmd.Execute); } @@ -104,12 +87,12 @@ public void TestLinkCatalogueToDatasetBadDataset() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(cmd.Execute); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); - var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository,"catalogue"), null, false); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "catalogue"), null, false); Assert.Throws(linkCmd.Execute); } - [Test] + [Test] public void TestLinkCatalogueToDatasetBadEverything() { var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), null, null, false); diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs index 02848abedf..30dcb53432 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs @@ -33,7 +33,7 @@ public void TestLinkColumnInfoToDataset() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), _c1, founddataset); Assert.DoesNotThrow(() => linkCmd.Execute()); var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects(); @@ -67,7 +67,7 @@ public void TestLinkColumnInfoToDatasetNotAll() var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), _c1, founddataset, false); Assert.DoesNotThrow(() => linkCmd.Execute()); var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().Where(ci => _cata1.CatalogueItems.Contains(ci)); @@ -87,7 +87,7 @@ public void TestLinkCatalogueToDatasetBadColumnInfo() { var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset"); Assert.DoesNotThrow(() => cmd.Execute()); - var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); + var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects().First(); var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), null, founddataset, false); Assert.Throws(() => linkCmd.Execute()); } diff --git a/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs b/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs index d57a3c5c19..5f9a5cd1c8 100644 --- a/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs +++ b/Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs @@ -38,6 +38,8 @@ public void Init() [TestCase(typeof(ExecuteCommandCreateNewCatalogueByImportingFile))] [TestCase(typeof(ExecuteCommandCreateNewCatalogueFromTableInfo))] [TestCase(typeof(ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration))] + [TestCase(typeof(ExecuteCommandUseTemplateCohortIdentificationConfiguration))] + [TestCase(typeof(ExecuteCommandCreateCohortIdentificationConfigurationTemplate))] [TestCase(typeof(ExecuteCommandCreateNewCohortFromCatalogue))] [TestCase(typeof(ExecuteCommandCreateNewCohortFromFile))] [TestCase(typeof(ExecuteCommandCreateNewCohortFromTable))] @@ -67,7 +69,6 @@ public void Init() [TestCase(typeof(ExecuteCommandAssociateCatalogueWithLoadMetadata))] [TestCase(typeof(ExecuteCommandAssociateCohortIdentificationConfigurationWithProject))] [TestCase(typeof(ExecuteCommandBulkImportTableInfos))] - [TestCase(typeof(ExecuteCommandChangeExtractability))] [TestCase(typeof(ExecuteCommandChangeExtractionCategory))] [TestCase(typeof(ExecuteCommandChangeLoadStage))] [TestCase(typeof(ExecuteCommandCheck))] diff --git a/Rdmp.Core.Tests/Curation/Integration/EncryptionTests.cs b/Rdmp.Core.Tests/Curation/Integration/EncryptionTests.cs index 9c8b30abfc..336e9fbc97 100644 --- a/Rdmp.Core.Tests/Curation/Integration/EncryptionTests.cs +++ b/Rdmp.Core.Tests/Curation/Integration/EncryptionTests.cs @@ -240,21 +240,4 @@ public void MigrationOfOldPasswordsTest() creds.DeleteInDatabase(); } } - - [Test] - public void PasswordTooLong() - { - if (RepositoryLocator.CatalogueRepository.EncryptionManager is PasswordEncryptionKeyLocation em && - !string.IsNullOrWhiteSpace(em.GetKeyFileLocation())) - Assert.Inconclusive( - "Could not run test because there is already an encryption key set up. Likely one that handles very long passwords"); - - var password = "a"; - for (var i = 0; i < 200; i++) - password += "a"; - - var ex = Assert.Throws(() => TestFreakyPasswordValues(password)); - Assert.That( - ex.Message, Is.EqualTo("The free text Value supplied to this class was too long to be encrypted (Length of string was 201)")); - } } \ No newline at end of file diff --git a/Rdmp.Core.Tests/Curation/Integration/PasswordEncryptionKeyLocationTests.cs b/Rdmp.Core.Tests/Curation/Integration/PasswordEncryptionKeyLocationTests.cs index ddfa9cec62..4502437df9 100644 --- a/Rdmp.Core.Tests/Curation/Integration/PasswordEncryptionKeyLocationTests.cs +++ b/Rdmp.Core.Tests/Curation/Integration/PasswordEncryptionKeyLocationTests.cs @@ -31,7 +31,6 @@ protected override void SetUp() public void NoKeyFileToStartWith() { var keyLocation = new PasswordEncryptionKeyLocation(CatalogueRepository); - //there shouldn't already be a key Assert.That(keyLocation.GetKeyFileLocation(), Is.Null); @@ -79,7 +78,6 @@ public void Encrypt() Console.WriteLine($"Encrypted (stock) is:{encrypter.Value}"); Console.WriteLine($"Decrypted (stock) is:{encrypter.GetDecryptedValue()}"); - var keyLocation = new PasswordEncryptionKeyLocation(CatalogueRepository); keyLocation.CreateNewKeyFile(Path.Combine(TestContext.CurrentContext.TestDirectory, "my.key")); var p = keyLocation.OpenKeyFile(); @@ -88,7 +86,6 @@ public void Encrypt() var s = CatalogueRepository.EncryptionManager.GetEncrypter(); var exception = Assert.Throws(() => s.Decrypt(encrypter.Value)); - Assert.That(exception.Message, Does.StartWith("Could not decrypt an encrypted string, possibly you are trying to decrypt it after having changed the PrivateKey ")); var encrypted = s.Encrypt(value); Console.WriteLine($"Encrypted (with key) is:{encrypted}"); diff --git a/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs b/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs index e6630495c7..0a275681f6 100644 --- a/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs +++ b/Rdmp.Core.Tests/Curation/ProjectspecificCatalogueTests.cs @@ -1,10 +1,12 @@ using Microsoft.Data.SqlClient; +using NPOI.OpenXmlFormats.Spreadsheet; using NUnit.Framework; using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.CommandLine.Interactive; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; +using Rdmp.Core.Providers; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Checks; using System; @@ -32,10 +34,13 @@ public void SetupTests(int projectCount = 0, int projectSpecificCount = 0, int e _catalogue.SaveToDatabase(); var _t1 = new TableInfo(CatalogueRepository, "T1"); _t1.SaveToDatabase(); + _activator.Publish(_t1); var _c1 = new ColumnInfo(CatalogueRepository, "PrivateIdentifierA", "varchar(10)", _t1); _c1.SaveToDatabase(); + _activator.Publish(_c1); var _ci1 = new CatalogueItem(CatalogueRepository, _catalogue, "PrivateIdentifierA"); _ci1.SaveToDatabase(); + _activator.Publish(_ci1); var _extractionInfo1 = new ExtractionInformation(CatalogueRepository, _ci1, _c1, _c1.ToString()) { Order = 123, @@ -43,18 +48,24 @@ public void SetupTests(int projectCount = 0, int projectSpecificCount = 0, int e IsExtractionIdentifier = true }; _extractionInfo1.SaveToDatabase(); + _activator.Publish(_extractionInfo1); if (projectCount > 0) { - _project1 = new Project(DataExportRepository, "Project1"); + _project1 = new Project(DataExportRepository, $"Project{projectCount}"); _project1.SaveToDatabase(); + _activator.Publish(_project1); + } if (projectCount > 1) { - _project2 = new Project(DataExportRepository, "Project1"); + _project2 = new Project(DataExportRepository, $"Project{projectCount}"); _project2.SaveToDatabase(); + _activator.Publish(_project2); + } _eds1 = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", _catalogue.ID).FirstOrDefault() ?? new ExtractableDataSet(_activator.RepositoryLocator.DataExportRepository, _catalogue); _eds1.SaveToDatabase(); + _activator.Publish(_eds1); if (projectSpecificCount > 0) { var cmd = new ExecuteCommandMakeCatalogueProjectSpecific(_activator); @@ -92,7 +103,7 @@ private void MakeProjectSpecific(Catalogue catalogue, Project project, List var cmd = new ExecuteCommandMakeCatalogueProjectSpecific(_activator,catalogue,project,force); cmd.SetTarget(project); cmd.SetTarget(catalogue); - if (shouldThrow) Assert.That(ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(_activator.RepositoryLocator.DataExportRepository,catalogue,project,projectIdsToIgnore??new List()), Is.False); + if (shouldThrow) Assert.That(ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(_activator.RepositoryLocator.DataExportRepository, catalogue,project,projectIdsToIgnore??new List()), Is.False); else Assert.DoesNotThrow(() => cmd.Execute()); } @@ -193,7 +204,7 @@ public void MakeSingleCatalogueProjectSpecificWhenAlreadyInOtherProjectExtractio public void MakeSingleCatalogueProjectSpecificWhenAlreadyInOtherProjectAllowExtractionTest() { SetupTests(2, 0, 1); - MakeProjectSpecific(_catalogue, _project1, new List() { _project2.ID }); + MakeProjectSpecific(_catalogue, _project1, new List() { _project1.ID }); MakeProjectSpecific(_catalogue, _project2, new List() { _project1.ID }); Assert.That(_activator.RepositoryLocator.CatalogueRepository.GetObjectByID(_catalogue.ID).IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository), Is.True); Assert.That(_activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", _catalogue.ID).First().Projects.Count, Is.EqualTo(2)); diff --git a/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs b/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs index 75f34dfb71..5bd22aeeff 100644 --- a/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs +++ b/Rdmp.Core.Tests/Curation/Unit/IMightBeReadOnlyTests.cs @@ -22,7 +22,7 @@ public void IsReadonly_AggregateFilterContainer() { //im probably an orphan var c = WhenIHaveA(); - Assert.That(c.ShouldBeReadOnly(out _), Is.False); + Assert.That(c.ShouldBeReadOnly(null,out _), Is.False); //now I am in a cic var cic = WhenIHaveA(); @@ -30,12 +30,12 @@ public void IsReadonly_AggregateFilterContainer() cic.CreateRootContainerIfNotExists(); cic.RootCohortAggregateContainer.AddChild(c.GetAggregate(), 0); - Assert.That(c.ShouldBeReadOnly(out _), Is.False); + Assert.That(c.ShouldBeReadOnly(null, out _), Is.False); cic.Frozen = true; Assert.Multiple(() => { - Assert.That(c.ShouldBeReadOnly(out var reason)); + Assert.That(c.ShouldBeReadOnly(null, out var reason)); Assert.That(reason, Is.EqualTo("fff is Frozen")); }); @@ -45,17 +45,17 @@ public void IsReadonly_AggregateFilterContainer() public void IsReadonly_ExtractionFilterContainer() { var c = WhenIHaveA(); - Assert.That(c.ShouldBeReadOnly(out _), Is.False); + Assert.That(c.ShouldBeReadOnly(null, out _), Is.False); var ec = c.GetSelectedDataSetIfAny().ExtractionConfiguration; - Assert.That(c.ShouldBeReadOnly(out _), Is.False); + Assert.That(c.ShouldBeReadOnly(null, out _), Is.False); ec.Name = "lll"; ec.IsReleased = true; Assert.Multiple(() => { - Assert.That(c.ShouldBeReadOnly(out var reason)); + Assert.That(c.ShouldBeReadOnly(null, out var reason)); Assert.That(reason, Is.EqualTo("lll has already been released")); }); @@ -66,17 +66,36 @@ public void IsReadonly_SpontaneousContainer() { var memoryrepo = new MemoryCatalogueRepository(); var c = new SpontaneouslyInventedFilterContainer(memoryrepo, null, null, FilterContainerOperation.AND); - Assert.That(c.ShouldBeReadOnly(out _), Is.False, + Assert.That(c.ShouldBeReadOnly(null, out _), Is.False, "Spont containers should never be in UI but let's not tell the programmer they shouldn't be edited"); } + [Test] + public void IsReadOnly_TemplateCohortIdentificationConfiguration() + { + var cic = WhenIHaveA(); + cic.IsTemplate = true; + cic.Frozen = true; + Assert.That(cic.ShouldBeReadOnly(null, out _), Is.True); + + } + + [Test] + public void IsReadOnly_TemplateCohortIdentificationConfiguration_Rename() + { + var cic = WhenIHaveA(); + cic.IsTemplate = true; + Assert.That(cic.ShouldBeReadOnly("ExecuteCommandRename", out _), Is.False); + + } + [Test] public void IsReadonly_AggregateFilter() { //im probably an orphan var f = WhenIHaveA(); - Assert.That(f.ShouldBeReadOnly(out _), Is.False); + Assert.That(f.ShouldBeReadOnly(null, out _), Is.False); //now I am in a cic var cic = WhenIHaveA(); @@ -84,12 +103,12 @@ public void IsReadonly_AggregateFilter() cic.CreateRootContainerIfNotExists(); cic.RootCohortAggregateContainer.AddChild(f.GetAggregate(), 0); - Assert.That(f.ShouldBeReadOnly(out _), Is.False); + Assert.That(f.ShouldBeReadOnly(null, out _), Is.False); cic.Frozen = true; Assert.Multiple(() => { - Assert.That(f.ShouldBeReadOnly(out var reason)); + Assert.That(f.ShouldBeReadOnly(null, out var reason)); Assert.That(reason, Is.EqualTo("fff is Frozen")); }); @@ -99,14 +118,14 @@ public void IsReadonly_AggregateFilter() public void IsReadonly_DeployedExtractionFilter() { var f = WhenIHaveA(); - Assert.That(f.ShouldBeReadOnly(out _), Is.False); + Assert.That(f.ShouldBeReadOnly(null, out _), Is.False); var ec = ((FilterContainer)f.FilterContainer).GetSelectedDataSetIfAny().ExtractionConfiguration; ec.Name = "lll"; ec.IsReleased = true; Assert.Multiple(() => { - Assert.That(f.ShouldBeReadOnly(out var reason)); + Assert.That(f.ShouldBeReadOnly(null, out var reason)); Assert.That(reason, Is.EqualTo("lll has already been released")); }); diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs index f1d4a30ac1..bbdf06a654 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs @@ -533,7 +533,7 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs() dt = destinationTable.GetDataTable(); - Assert.That(dt.Rows, Has.Count.EqualTo(2)); + Assert.That(dt.Rows, Has.Count.EqualTo(3)); } [Test] @@ -802,7 +802,7 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK() dt = destinationTable.GetDataTable(); - Assert.That(dt.Rows, Has.Count.EqualTo(2)); + Assert.That(dt.Rows, Has.Count.EqualTo(3)); } [Test] @@ -1061,7 +1061,7 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK() dt = destinationTable.GetDataTable(); - Assert.That(dt.Rows, Has.Count.EqualTo(2)); + Assert.That(dt.Rows, Has.Count.EqualTo(3)); } [Test] @@ -1209,6 +1209,8 @@ public void ExtractToDatabaseUseTriggers() var argumentDbNamePattern = destinationArguments.Single(static a => a.Name == "DatabaseNamingPattern"); var argumentTblNamePattern = destinationArguments.Single(static a => a.Name == "TableNamingPattern"); var reExtract = destinationArguments.Single(static a => a.Name == "AppendDataIfTableExists"); + var argUseArchiveTrigger = destinationArguments.Single(static a => a.Name == "UseArchiveTrigger"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); var extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) { @@ -1224,6 +1226,8 @@ public void ExtractToDatabaseUseTriggers() argumentDbNamePattern.SaveToDatabase(); argumentTblNamePattern.SetValue("$c_$d"); argumentTblNamePattern.SaveToDatabase(); + argUseArchiveTrigger.SetValue(true); + argUseArchiveTrigger.SaveToDatabase(); reExtract.SetValue(true); reExtract.SaveToDatabase(); diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs new file mode 100644 index 0000000000..34d50b3d06 --- /dev/null +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs @@ -0,0 +1,777 @@ +// Copyright (c) The University of Dundee 2018-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using FAnsi; +using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; +using Microsoft.Data.SqlClient; +using NUnit.Framework; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands.CatalogueCreationCommands; +using Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands; +using Rdmp.Core.CommandLine.DatabaseCreation; +using Rdmp.Core.CommandLine.Options; +using Rdmp.Core.CommandLine.Runners; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Aggregation; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.Curation.Data.Pipelines; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; +using Rdmp.Core.DataExport.DataExtraction.Pipeline.Sources; +using Rdmp.Core.DataFlowPipeline; +using Rdmp.Core.DataLoad.Triggers; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; +using SynthEHR; +using SynthEHR.Datasets; +using System; +using System.Data; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using Tests.Common; +using Tests.Common.Scenarios; +using TypeGuesser; + +namespace Rdmp.Core.Tests.DataExport.DataExtraction; + +public class ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest : DatabaseTests +{ + + private FileInfo CreateFileInForLoading(string filename, int rows, Random r) + { + var fi = new FileInfo(Path.Combine(Path.GetTempPath(), Path.GetFileName(filename))); + + var demog = new Demography(r); + var people = new PersonCollection(); + people.GeneratePeople(500, r); + + demog.GenerateTestDataFile(people, fi, rows); + + return fi; + } + + [Test] + public void SQLServerDestinationWithTriggersNoUpdate() { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); + + ec.SaveToDatabase(); + + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger= destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + var hicLoadID = dt.Rows[0].ItemArray[38]; + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[38], Is.EqualTo(hicLoadID)); + + } + + [Test] + public void SQLServerDestinationWithTriggersAndUpdate() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); + + ec.SaveToDatabase(); + + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 2"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + var hicLoadID = dt.Rows[0].ItemArray[38]; + + //update source here + var server = catalogue.GetDistinctLiveDatabaseServer(ReusableLibraryCode.DataAccess.DataAccessContext.InternalDataProcessing, false, out var dap); + if (server.Exists()) + { + var fdb = server.ExpectDatabase(catalogue.CatalogueItems.First().ColumnInfo.TableInfo.Database); + var ftbl = fdb.ExpectTable("bob"); + using (var con = server.GetConnection()) + { + con.Open(); + var sql = $"UPDATE {ftbl.GetFullyQualifiedName()} set notes = 'JOHN'"; + var cmd1 = server.GetCommand(sql, con); + cmd1.ExecuteNonQuery(); + } + + } + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Rows[0].ItemArray[38], Is.Not.EqualTo(hicLoadID)); + } + [Test] + public void SQLServerDestinationWithTriggersNoPKs() { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = false; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); + + ec.SaveToDatabase(); + + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 1"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + var hicLoadID = dt.Rows[0].ItemArray[38]; + + //update source here + var server = catalogue.GetDistinctLiveDatabaseServer(ReusableLibraryCode.DataAccess.DataAccessContext.InternalDataProcessing, false, out var dap); + if (server.Exists()) + { + var fdb = server.ExpectDatabase(catalogue.CatalogueItems.First().ColumnInfo.TableInfo.Database); + var ftbl = fdb.ExpectTable("bob"); + using (var con = server.GetConnection()) + { + con.Open(); + var sql = $"UPDATE {ftbl.GetFullyQualifiedName()} set notes = 'JOHN'"; + var cmd1 = server.GetCommand(sql, con); + cmd1.ExecuteNonQuery(); + } + + } + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(2)); + } + +} \ No newline at end of file diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs index ffe63b1da1..95382b5950 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs @@ -1844,7 +1844,7 @@ public void DataTableUploadClashUpdateDropColumnWithTrigger() using var resultDt = table.GetDataTable(); Assert.That(resultDt.Rows, Has.Count.EqualTo(1)); Assert.That(resultDt.Rows[0].ItemArray[0], Is.EqualTo("Fish")); - Assert.That(resultDt.Rows[0].ItemArray[1], Is.EqualTo(string.Empty)); + Assert.That(resultDt.Rows[0].ItemArray[1].ToString(), Is.EqualTo(String.Empty)); table = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests_Archive"); using var resultDt2 = table.GetDataTable(); Assert.That(resultDt2.Rows, Has.Count.EqualTo(1)); diff --git a/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs b/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs index c2a6ed9397..abe136451c 100644 --- a/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs +++ b/Rdmp.Core.Tests/Providers/SearchablesMatchScorerTests.cs @@ -124,19 +124,6 @@ public void TestScoringCatalogueFlag_IsDeprecated(bool hasFlag, bool shouldShow, }, expectedResult); } - [TestCase(true, true, true)] - [TestCase(true, false, false)] - [TestCase(false, true, true)] - [TestCase(false, false, true)] - public void TestScoringCatalogueFlag_IsColdStorage(bool hasFlag, bool shouldShow, bool expectedResult) - { - TestScoringFlag((c, eds) => - { - c.IsColdStorageDataset = hasFlag; - UserSettings.ShowColdStorageCatalogues = shouldShow; - }, expectedResult); - } - [TestCase(true, true, true)] [TestCase(true, false, false)] [TestCase(false, true, true)] @@ -150,20 +137,6 @@ public void TestScoringCatalogueFlag_IsInternalDataset(bool hasFlag, bool should }, expectedResult); } - [TestCase(true, true, true)] - [TestCase(true, false, false)] - [TestCase(false, true, true)] - [TestCase(false, false, true)] - public void TestScoringCatalogueFlag_IsExtractable(bool notExtractable, bool shouldShow, bool expectedResult) - { - TestScoringFlag((c, eds) => - { - if (notExtractable) eds.DeleteInDatabase(); - - UserSettings.ShowNonExtractableCatalogues = shouldShow; - }, expectedResult); - } - [TestCase(true, true, true)] [TestCase(true, false, false)] [TestCase(false, true, true)] @@ -194,10 +167,8 @@ private void TestScoringFlag(Action setter, bool // // So set all to false to except the condition we are testing UserSettings.ShowDeprecatedCatalogues = false; - UserSettings.ShowNonExtractableCatalogues = false; UserSettings.ShowProjectSpecificCatalogues = false; UserSettings.ShowInternalCatalogues = false; - UserSettings.ShowColdStorageCatalogues = false; var c = WhenIHaveA(); c.Name = "Bunny"; diff --git a/Rdmp.Core.Tests/Reports/CustomMetadataReportTests.cs b/Rdmp.Core.Tests/Reports/CustomMetadataReportTests.cs index bf2a9a163a..8223b48985 100644 --- a/Rdmp.Core.Tests/Reports/CustomMetadataReportTests.cs +++ b/Rdmp.Core.Tests/Reports/CustomMetadataReportTests.cs @@ -1384,7 +1384,6 @@ public void TestAllSubs_Catalogue() $Geographical_coverage $Granularity $ID -$IsColdStorageDataset $IsDeprecated $IsInternalDataset $Last_revision_date diff --git a/Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs b/Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs new file mode 100644 index 0000000000..2517b2e17e --- /dev/null +++ b/Rdmp.Core.Tests/Reports/ExtractionTime/DatasetVariableReportGeneratorTests.cs @@ -0,0 +1,25 @@ +using System.IO; +using NUnit.Framework; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.DataExport.DataExtraction.Pipeline; +using Rdmp.Core.Reports.ExtractionTime; +using Tests.Common.Scenarios; + +namespace Rdmp.Core.Tests.Reports.ExtractionTime +{ + internal class DatasetVariableReportGeneratorTests: TestsRequiringAnExtractionConfiguration + { + [Test] + public void Test_DatasetVariableReportGenerator_Creation() + { + this.Execute(out ExtractionPipelineUseCase pipelineUseCase,out var results); + var report = new DatasetVariableReportGenerator(pipelineUseCase); + report.GenerateDatasetVariableReport(); + var filename = Path.Join( + pipelineUseCase.Destination.DirectoryPopulated.FullName, + $"{pipelineUseCase.Destination.GetFilename()}Variables.csv" + ); + Assert.That(File.Exists(filename)); + } + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 7d01c6ea01..e72eeece74 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -19,6 +19,7 @@ using Rdmp.Core.Curation.Data.Cache; using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Curation.Data.Defaults; using Rdmp.Core.Curation.Data.Pipelines; using Rdmp.Core.Curation.Data.Referencing; @@ -41,6 +42,7 @@ using Rdmp.Core.ReusableLibraryCode.DataAccess; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; +using static Microsoft.IO.RecyclableMemoryStreamManager; namespace Rdmp.Core.CommandExecution; @@ -129,24 +131,34 @@ public IEnumerable CreateCommands(object o) }; if (!isApiCall) { - yield return new ExecuteCommandChangeExtractability(_activator, c) - { - Weight = -99.0010f, - SuggestedCategory = Extraction - }; + yield return c.IsInternalDataset ? + new ExecuteCommandMakeCatalogueNotInternal(_activator, c) + { + Weight = -99.0008f, + SuggestedCategory = Extraction + } : new ExecuteCommandMakeCatalogueInternal(_activator, c) + { + Weight = -99.0008f, + SuggestedCategory = Extraction + }; + + - yield return new ExecuteCommandMakeCatalogueProjectSpecific(_activator, c, null,false) + yield return new ExecuteCommandMakeCatalogueProjectSpecific(_activator, c, null, false) { Weight = -99.0009f, SuggestedCategory = Extraction }; - yield return new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, c,null) - { - Weight = -99.0009f, - SuggestedCategory = Extraction, - OverrideCommandName="Remove Project Specific Catalogue from a Project" - }; + if (c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) + { + yield return new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, c, null) + { + Weight = -99.0009f, + SuggestedCategory = Extraction, + OverrideCommandName = "Remove Project Specific Catalogue from a Project" + }; + } yield return new ExecuteCommandSetExtractionIdentifier(_activator, c, null, null) { Weight = -99.0008f, @@ -261,7 +273,6 @@ public IEnumerable CreateCommands(object o) yield return new ExecuteCommandCreateNewFilter(_activator, ac) { OfferCatalogueFilters = true, - OfferCohortCatalogueFilters=true, SuggestedCategory = Add, OverrideCommandName = "Existing Filter" }; @@ -331,7 +342,6 @@ public IEnumerable CreateCommands(object o) yield return new ExecuteCommandCreateNewFilter(_activator, container, null) { OfferCatalogueFilters = true, - OfferCohortCatalogueFilters = true, SuggestedCategory = Add, OverrideCommandName = "Existing Filter" }; @@ -457,34 +467,51 @@ public IEnumerable CreateCommands(object o) { if (pcic != null) cic = pcic.CohortIdentificationConfiguration; - var commit = - new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration(_activator, null) - { - OverrideCommandName = "Commit Cohort", - Weight = -99.8f - }.SetTarget(cic); - if (pcic != null) commit.SetTarget((DatabaseEntity)pcic.Project); - yield return commit; yield return new ExecuteCommandViewData(_activator, cic, ViewType.All, null, true) { Weight = -99.7f }; yield return new ExecuteCommandViewData(_activator, cic, ViewType.All, null, false) { Weight = -99.6f }; - yield return new ExecuteCommandFreezeCohortIdentificationConfiguration(_activator, cic, !cic.Frozen) - { Weight = -50.5f }; - yield return new ExecuteCommandCreateHoldoutLookup(_activator, cic) - { Weight = -50.5f }; + if (cic != null && cic.IsTemplate) + { + yield return new ExecuteCommandUseTemplateCohortIdentificationConfiguration(_activator, cic) + { + Weight = -50.5f + }; + } + else + { + var commit = + new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration(_activator, null) + { + OverrideCommandName = "Commit Cohort", + Weight = -99.8f + }.SetTarget(cic); + if (pcic != null) commit.SetTarget((DatabaseEntity)pcic.Project); + + yield return commit; + yield return new ExecuteCommandCreateCohortIdentificationConfigurationTemplate(_activator, cic) + { + Weight = -50.5f + }; + yield return new ExecuteCommandFreezeCohortIdentificationConfiguration(_activator, cic, !cic.Frozen) + { Weight = -50.5f }; + yield return new ExecuteCommandCreateHoldoutLookup(_activator, cic) + { Weight = -50.5f }; + var clone = new ExecuteCommandCloneCohortIdentificationConfiguration(_activator) + { Weight = -50.4f, OverrideCommandName = "Clone" }.SetTarget(cic); + if (pcic != null) clone.SetTarget((DatabaseEntity)pcic.Project); + yield return clone; + + yield return new ExecuteCommandSetQueryCachingDatabase(_activator, cic) + { Weight = -50.4f, OverrideCommandName = "Change Query Cache" }; + } + - var clone = new ExecuteCommandCloneCohortIdentificationConfiguration(_activator) - { Weight = -50.4f, OverrideCommandName = "Clone" }.SetTarget(cic); - if (pcic != null) clone.SetTarget((DatabaseEntity)pcic.Project); - yield return clone; //associate with project yield return new ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(_activator) { Weight = -50.3f, OverrideCommandName = "Associate with Project" }.SetTarget(cic); - yield return new ExecuteCommandSetQueryCachingDatabase(_activator, cic) - { Weight = -50.4f, OverrideCommandName = "Change Query Cache" }; } if (Is(o, out AllGovernanceNode _)) @@ -696,7 +723,6 @@ public IEnumerable CreateCommands(object o) yield return new ExecuteCommandCreateNewFilter(_activator, sds) { OfferCatalogueFilters = true, - OfferCohortCatalogueFilters = true, OverrideCommandName = "Existing Filter (copy of)", SuggestedCategory = Add }; @@ -918,42 +944,48 @@ public IEnumerable CreateCommands(object o) if (Is(o, out CohortAggregateContainer cohortAggregateContainer)) { - yield return new ExecuteCommandAddCatalogueToCohortIdentificationSetContainer(_activator, - cohortAggregateContainer, null, null) - { SuggestedCategory = Add, OverrideCommandName = "Catalogue" }; - yield return new ExecuteCommandAddCohortSubContainer(_activator, cohortAggregateContainer) - { SuggestedCategory = Add, OverrideCommandName = "Sub Container" }; - yield return new ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer(_activator, - cohortAggregateContainer, true) - { SuggestedCategory = Add, OverrideCommandName = "Existing Cohort Set (copy of)" }; - yield return new ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer(_activator, - cohortAggregateContainer, false) - { SuggestedCategory = Add, OverrideCommandName = "Aggregate" }; - yield return new ExecuteCommandImportCohortIdentificationConfiguration(_activator, null, + var associatedCIC = cohortAggregateContainer.GetCohortIdentificationConfiguration(); + if (!associatedCIC.Frozen) + { + yield return new ExecuteCommandAddCatalogueToCohortIdentificationSetContainer(_activator, + cohortAggregateContainer, null, null) + { SuggestedCategory = Add, OverrideCommandName = "Catalogue" }; + yield return new ExecuteCommandAddCohortSubContainer(_activator, cohortAggregateContainer) + { SuggestedCategory = Add, OverrideCommandName = "Sub Container" }; + yield return new ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer(_activator, + cohortAggregateContainer, true) + { SuggestedCategory = Add, OverrideCommandName = "Existing Cohort Set (copy of)" }; + yield return new ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer(_activator, + cohortAggregateContainer, false) + { SuggestedCategory = Add, OverrideCommandName = "Aggregate" }; + yield return new ExecuteCommandImportCohortIdentificationConfiguration(_activator, null, + cohortAggregateContainer) + { SuggestedCategory = Add, OverrideCommandName = "Existing Cohort Builder Query (copy of)" }; + + //Set Operation + yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, + SetOperation.UNION) + { SuggestedCategory = SetContainerOperation, OverrideCommandName = "UNION" }; + yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, + SetOperation.EXCEPT) + { SuggestedCategory = SetContainerOperation, OverrideCommandName = "EXCEPT" }; + yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, + SetOperation.INTERSECT) + { SuggestedCategory = SetContainerOperation, OverrideCommandName = "INTERSECT" }; + + yield return new ExecuteCommandUnMergeCohortIdentificationConfiguration(_activator, cohortAggregateContainer) - { SuggestedCategory = Add, OverrideCommandName = "Existing Cohort Builder Query (copy of)" }; - - //Set Operation - yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, - SetOperation.UNION) - { SuggestedCategory = SetContainerOperation, OverrideCommandName = "UNION" }; - yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, - SetOperation.EXCEPT) - { SuggestedCategory = SetContainerOperation, OverrideCommandName = "EXCEPT" }; - yield return new ExecuteCommandSetContainerOperation(_activator, cohortAggregateContainer, - SetOperation.INTERSECT) - { SuggestedCategory = SetContainerOperation, OverrideCommandName = "INTERSECT" }; - - yield return new ExecuteCommandUnMergeCohortIdentificationConfiguration(_activator, - cohortAggregateContainer) - { OverrideCommandName = "Separate Cohort Builder Query" }; + { OverrideCommandName = "Separate Cohort Builder Query" }; + } } if (Is(o, out IDisableable disable)) + //todo this calls the db yield return new ExecuteCommandDisableOrEnable(_activator, disable); // If the root object is deletable offer deleting if (Is(o, out IDeleteable deletable)) + //todo this calls the db yield return new ExecuteCommandDelete(_activator, deletable) { SuggestedShortcut = "Delete" }; if (Is(o, out ReferenceOtherObjectDatabaseEntity reference)) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs new file mode 100644 index 0000000000..966e21b400 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateCohortIdentificationConfigurationTemplate.cs @@ -0,0 +1,91 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.ReusableLibraryCode.Checks; +using System; +using System.Linq; +using System.Reflection.Metadata.Ecma335; + +namespace Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands +{ + public class ExecuteCommandCreateCohortIdentificationConfigurationTemplate : BasicCommandExecution, IAtomicCommandWithTarget + { + private CohortIdentificationConfiguration _cic; + private IMapsDirectlyToDatabaseTable _selectedProject; + private readonly IBasicActivateItems _activator; + public ExecuteCommandCreateCohortIdentificationConfigurationTemplate(IBasicActivateItems activator, CohortIdentificationConfiguration cic) : base(activator) + { + _activator = activator; + _cic = cic; + } + + private string GenerateTemplateName(string name) + { + if (name.EndsWith(" (Clone)")) name = name.Substring(0, name.Length - 8); + if (name.EndsWith(" Template")) return name; + return name + " Template"; + } + + public override void Execute() + { + var associations = _activator.RepositoryLocator.DataExportRepository.GetAllObjects(); + var projectAssociations = associations.Where(a => a.CohortIdentificationConfiguration_ID == _cic.ID).ToList(); + base.Execute(); + var clone = _cic.CreateClone(ThrowImmediatelyCheckNotifier.Quiet); + clone.IsTemplate = true; + clone.Freeze(); + clone.Name = GenerateTemplateName(clone.Name); + clone.SaveToDatabase(); + + if (_activator.IsInteractive && projectAssociations.Any()) + { + if (projectAssociations.Count > 1) + { + //multiple, make them pick + if (_activator.YesNo("This Cohort Configuration is associated with multiple Projects. Would you like to associate this Template with one of them?", "Associate with a Project")) + { + _selectedProject = _activator.SelectOne("Select a Project to associate this Template with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + } + } + else if (projectAssociations.Count == 1) + { + + //ask them if they want to use this one + if (_activator.YesNo($"This cohort configuration is already associated with the {projectAssociations.First().Project.Name} project. Would you like to associate this Template with this project?", "Use Existing Project")) + { + _selectedProject = projectAssociations.First().Project; + } + } + else + { + //ask if they want to use it in a project + if (_activator.YesNo("Would you like to associate this Template with a project?", "Associate with a Project")) + { + _selectedProject = _activator.SelectOne("Select a Project to associate this Template with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + } + } + } + if (_selectedProject != null) + { + var cmd = new ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(_activator); + cmd.SetTarget(clone); + cmd.SetTarget((Project)_selectedProject); + cmd.Execute(); + } + Publish(clone); + Emphasise(clone); + } + + public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) + { + if (target is not CohortIdentificationConfiguration && target is not Project) + { + throw new Exception("Provided database entity was not a CohortIdentificationConfiguration or a Project."); + } + if (target is Project p) _selectedProject = p; + else if (target is CohortIdentificationConfiguration c) _cic = c; + return this; + } + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs index 39dba10eb1..8e92bf6c98 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs @@ -83,11 +83,19 @@ public override void Execute() } } - if (Project == null && BasicActivator.CoreChildProvider is DataExportChildProvider dx) + + IProject currentProj = null; + ProjectCohortIdentificationConfigurationAssociation[] projAssociations = Array.Empty(); + if (BasicActivator.CoreChildProvider is DataExportChildProvider dx) { - var projAssociations = dx.AllProjectAssociatedCics + projAssociations = dx.AllProjectAssociatedCics .Where(c => c.CohortIdentificationConfiguration_ID == cic.ID).ToArray(); - if (projAssociations.Length == 1) Project = projAssociations[0].Project; + if (projAssociations.Length > 0) + { + currentProj = Project != null ? Project : projAssociations.Length == 1 ? projAssociations[0].Project : null; + Project = BasicActivator.CohortCommitProjectSelect(currentProj, BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjects().ToArray()); + if (Project is null) return; + } } var auditLogBuilder = new ExtractableCohortAuditLogBuilder(); @@ -95,7 +103,10 @@ public override void Execute() //user choose to cancel the cohort creation request dialogue if (request == null) + { + Project = null; return; + } request.CohortIdentificationConfiguration = cic; @@ -103,7 +114,16 @@ public override void Execute() configureAndExecute.PipelineExecutionFinishedsuccessfully += (s, u) => OnImportCompletedSuccessfully(cic); - configureAndExecute.Run(BasicActivator.RepositoryLocator, null, null, null); + var result = configureAndExecute.Run(BasicActivator.RepositoryLocator, null, null, null); + if (result ==0 && currentProj != null && currentProj.ID != Project.ID) + { + //move cic to new project + var oldAssociation = projAssociations.Where(a => a.Project_ID == currentProj.ID).FirstOrDefault(); + if (oldAssociation != null) + { + oldAssociation.DeleteInDatabase(); + } + } } private void OnImportCompletedSuccessfully(CohortIdentificationConfiguration cic) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs new file mode 100644 index 0000000000..643bc692fa --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandUseTemplateCohortIdentificationConfiguration.cs @@ -0,0 +1,99 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.Repositories; +using Rdmp.Core.ReusableLibraryCode.Checks; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands +{ + public class ExecuteCommandUseTemplateCohortIdentificationConfiguration : BasicCommandExecution, IAtomicCommandWithTarget + { + private CohortIdentificationConfiguration _cic; + private IMapsDirectlyToDatabaseTable _selectedProject; + private readonly IBasicActivateItems _activator; + public ExecuteCommandUseTemplateCohortIdentificationConfiguration(IBasicActivateItems activator, CohortIdentificationConfiguration cic) : base(activator) + { + _activator = activator; + _cic = cic; + } + + private string RenameTemplateForUse(string name) + { + if (name.EndsWith(" (Clone)")) name = name.Substring(0, name.Length - 8); + if (name.EndsWith("Template")) + { + name = name.Substring(0,name.Length - 8); + } + while(name.EndsWith(" ")) + { + name = name.Substring(0,name.Length - 1); + } + return name; + } + + public override void Execute() + { + var associations = _activator.RepositoryLocator.DataExportRepository.GetAllObjects(); + var projectAssociations = associations.Where(a => a.CohortIdentificationConfiguration_ID == _cic.ID).ToList(); + if (_activator.IsInteractive && projectAssociations.Any()) + { + if (projectAssociations.Count > 1) + { + //multiple, make them pick + if (_activator.YesNo("This Template is associated with multiple Projects. Would you like to associate this cohort configuration with one of them?", "Associate with a Project")) + { + _selectedProject = _activator.SelectOne("Select a Project to associate this cohort with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + } + } + else if (projectAssociations.Count == 1) + { + + //ask them if they want to use this one + if (_activator.YesNo($"This template is already associated with the {projectAssociations.First().Project.Name} project. Would you like to associate this cohort configuration with this project?", "Use Existing Project")) + { + _selectedProject = projectAssociations.First().Project; + } + } + else + { + //ask if they want to use it in a project + if (_activator.YesNo("Would you like to associate this cohort configuration with a project?", "Associate with a Project")) + { + _selectedProject = _activator.SelectOne("Select a Project to associate this cohort with", _activator.RepositoryLocator.DataExportRepository.GetAllObjects()); + } + } + } + base.Execute(); + var clone = _cic.CreateClone(ThrowImmediatelyCheckNotifier.Quiet); + clone.IsTemplate = false; + clone.Name = RenameTemplateForUse(clone.Name); + clone.SaveToDatabase(); + if (_selectedProject != null) + { + var cmd = new ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(_activator); + cmd.SetTarget(clone); + cmd.SetTarget((Project)_selectedProject); + cmd.Execute(); + } + Publish(clone); + Emphasise(clone); + } + + public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) + { + if ((target is not CohortIdentificationConfiguration && target is not Project) || (target is CohortIdentificationConfiguration cic && !cic.IsTemplate)) + { + throw new Exception("Provided database entity was not a CohortIdentificationConfiguration or a Project."); + } + if (target is Project p) _selectedProject = p; + else if (target is CohortIdentificationConfiguration c) _cic = c; + return this; + } + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer.cs index a589b93d46..4bcaa678ac 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContainer.cs @@ -44,7 +44,7 @@ private ExecuteCommandAddAggregateConfigurationToCohortIdentificationSetContaine { _targetCohortAggregateContainer = targetCohortAggregateContainer; - if (targetCohortAggregateContainer.ShouldBeReadOnly(out var reason)) + if (targetCohortAggregateContainer.ShouldBeReadOnly(this.GetType().Name,out var reason)) SetImpossible(reason); UseTripleDotSuffix = true; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs index 3e70c2c720..114cd2974e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddCatalogueToCohortIdentificationSetContainer.cs @@ -4,11 +4,13 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using System.Collections.Generic; using System.Linq; using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.Repositories.Construction; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; @@ -49,7 +51,7 @@ public ExecuteCommandAddCatalogueToCohortIdentificationSetContainer(IBasicActiva _targetCohortAggregateContainer = targetCohortAggregateContainer; - if (targetCohortAggregateContainer.ShouldBeReadOnly(out var reason)) + if (targetCohortAggregateContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); if (catalogue != null) @@ -111,18 +113,33 @@ public override void Execute() // if user hasn't picked a Catalogue yet if (_catalogueCombineable == null) { + var cic = _targetCohortAggregateContainer.GetCohortIdentificationConfiguration(); + List associatedProjectCataloguesIDs= new(); + var pcica = BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(pcica => pcica.CohortIdentificationConfiguration_ID == cic.ID).FirstOrDefault(); + if(pcica is not null) + { + associatedProjectCataloguesIDs = pcica.Project.GetAllProjectCatalogues().Select(c => c.ID).ToList(); + } if (!BasicActivator.SelectObjects(new DialogArgs { WindowTitle = "Add Catalogue(s) to Container", TaskDescription = $"Choose which Catalogues to add to the cohort container '{_targetCohortAggregateContainer.Name}'. Catalogues must have at least one IsExtractionIdentifier column." - }, BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects(), out var selected)) + }, BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects().Where(c => !c.IsInternalDataset &&(!c.IsProjectSpecific(BasicActivator.RepositoryLocator.DataExportRepository) || associatedProjectCataloguesIDs.Contains(c.ID))).ToArray(), out var selected)) // user didn't pick one return; // for each catalogue they picked foreach (var catalogue in selected) { + if(BasicActivator.IsInteractive && catalogue.IsDeprecated) + { + var confirmDeprecatedUser = BasicActivator.YesNo($"{catalogue.Name} is marked as deprecated. Are you sure you wish to use it?", "Confirm use of Deprecated Catalogue"); + if (!confirmDeprecatedUser) + { + continue; + } + } var combineable = new CatalogueCombineable(catalogue); UpdateIsImpossibleFor(combineable); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs index d395395fa6..25880568e5 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddDatasetsToConfiguration.cs @@ -4,6 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using System; using System.Linq; using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.DataExport.Data; @@ -62,7 +63,17 @@ public ExecuteCommandAddDatasetsToConfiguration(IBasicActivateItems itemActivato var _importableDataSets = childProvider.ExtractableDataSets.Except(_datasets) //where it can be used in any Project OR this project only - .Where(ds => !ds.Projects.Any()|| ds.Projects.Select(p => p.ID).Contains(targetExtractionConfiguration.Project_ID)) + .Where(ds => + { + try + { + return (!ds.Projects.Any() || ds.Projects.Select(p => p.ID).Contains(targetExtractionConfiguration.Project_ID)) && !ds.Catalogue.IsInternalDataset; + } + catch (Exception) + { + return false; + } + }) .ToArray(); SetExtractableDataSets(true, _importableDataSets); @@ -97,8 +108,10 @@ public override void Execute() }, _toadd.Cast().ToArray(), out var selected)) return; - foreach (var ds in selected) + foreach (var ds in selected.Where(ds => !ds.Catalogue.IsDeprecated || (BasicActivator.IsInteractive && ds.Catalogue.IsDeprecated && YesNo($"{ds.Catalogue.Name} is deprecated. Are you sure you wish to extract it?", "Confirm use of Deprecated Catalogue")))) + { _targetExtractionConfiguration.AddDatasetToConfiguration(ds); + } } else { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs index 29bb64882d..87e5be28e3 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewAggregateGraph.cs @@ -25,8 +25,7 @@ public ExecuteCommandAddNewAggregateGraph(IBasicActivateItems activator, Catalog { _catalogue = catalogue; _name = name; - if (_catalogue != null && _catalogue.GetAllExtractionInformation(ExtractionCategory.Any) - .All(ei => ei.ColumnInfo == null)) + if (_catalogue != null && _catalogue.GetAllExtractionInformation(ExtractionCategory.Any).All(ei => ei.ColumnInfo == null)) SetImpossible("Catalogue has no extractable columns"); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs index 7d9580e68f..bf04ff3d31 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddNewCatalogueItem.cs @@ -23,6 +23,7 @@ public class ExecuteCommandAddNewCatalogueItem : BasicCommandExecution, IAtomicC private readonly Catalogue _catalogue; private readonly ColumnInfo[] _columnInfos; private readonly HashSet _existingColumnInfos; + private readonly IBasicActivateItems _activator; /// /// The category to assign for newly created . @@ -33,12 +34,14 @@ public class ExecuteCommandAddNewCatalogueItem : BasicCommandExecution, IAtomicC public ExecuteCommandAddNewCatalogueItem(IBasicActivateItems activator, Catalogue catalogue, ColumnInfoCombineable colInfo) : this(activator, catalogue, colInfo.ColumnInfos) { + _activator = activator; } [UseWithObjectConstructor] public ExecuteCommandAddNewCatalogueItem(IBasicActivateItems activator, Catalogue catalogue, params ColumnInfo[] columnInfos) : base(activator) { + _activator = activator; _catalogue = catalogue; _existingColumnInfos = GetColumnInfos(_catalogue); _columnInfos = columnInfos; @@ -48,11 +51,11 @@ public ExecuteCommandAddNewCatalogueItem(IBasicActivateItems activator, Catalogu SetImpossible("ColumnInfo(s) are already in Catalogue"); } - private static HashSet GetColumnInfos(ICatalogue catalogue) + private HashSet GetColumnInfos(ICatalogue catalogue) { return catalogue == null ? null - : new HashSet(catalogue.CatalogueItems.Select(static ci => ci.ColumnInfo_ID) + : new HashSet(_activator.CoreChildProvider.AllCatalogueItems.Where(ci => ci.Catalogue_ID == catalogue.ID).Select(static ci => ci.ColumnInfo_ID) .Where(static col => col.HasValue).Select(static v => v.Value).Distinct()); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAssociateCohortIdentificationConfigurationWithProject.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAssociateCohortIdentificationConfigurationWithProject.cs index 23fe289151..35e3e007ea 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAssociateCohortIdentificationConfigurationWithProject.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAssociateCohortIdentificationConfigurationWithProject.cs @@ -9,6 +9,7 @@ using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.DataExport.Data; using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.Providers; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; @@ -28,8 +29,7 @@ public ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(IBasi if (!activator.CoreChildProvider.AllCohortIdentificationConfigurations.Any()) SetImpossible("There are no Cohort Identification Configurations yet"); - _existingAssociations = BasicActivator.RepositoryLocator.DataExportRepository - .GetAllObjects(); + _existingAssociations = ((DataExportChildProvider)activator.CoreChildProvider).AllProjectAssociatedCics; } public override string GetCommandHelp() => diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs deleted file mode 100644 index f27befd4a3..0000000000 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) The University of Dundee 2018-2019 -// This file is part of the Research Data Management Platform (RDMP). -// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -// You should have received a copy of the GNU General Public License along with RDMP. If not, see . - -using System.Linq; -using Rdmp.Core.Curation.Data; -using Rdmp.Core.DataExport.Data; -using Rdmp.Core.Icons.IconProvision; -using Rdmp.Core.Providers; -using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; -using SixLabors.ImageSharp; -using SixLabors.ImageSharp.PixelFormats; - -namespace Rdmp.Core.CommandExecution.AtomicCommands; - -public sealed class ExecuteCommandChangeExtractability : BasicCommandExecution -{ - private readonly Catalogue _catalogue; - private readonly bool _markExtractable; - - public ExecuteCommandChangeExtractability(IBasicActivateItems activator, Catalogue catalogue, - bool? explicitExtractability = null) : base(activator) - { - _catalogue = catalogue; - var status = catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository); - if (status == null) - { - SetImpossible( - "We don't know whether Catalogue is extractable or not (possibly no Data Export database is available)"); - return; - } - - if (status.IsProjectSpecific) - { - SetImpossible( - "Cannot change the extractability because it is configured as a 'Project Specific Catalogue'"); - return; - } - - // mark it extractable true/false as passed in constructor or just flip its state - _markExtractable = explicitExtractability ?? !status.IsExtractable; - } - - public override string GetCommandName() => _markExtractable ? "Mark Extractable" : "Mark Not Extractable"; - - public override string GetCommandHelp() => - !_markExtractable - ? "Prevent dataset from being released in Project extracts. This fails if it is already part of any ExtractionConfigurations" - : @"Enable dataset linkage\extraction in Project extracts. This requires that at least one column be marked IsExtractionIdentifier"; - - public override Image GetImage(IIconProvider iconProvider) => - iconProvider.GetImage(RDMPConcept.ExtractableDataSet, _markExtractable ? OverlayKind.Add : OverlayKind.Delete); - - public override void Execute() - { - base.Execute(); - - if (_markExtractable) - { - if (_catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository).IsExtractable) - { - Show($"{_catalogue} is already extractable"); - } - else - { - new ExtractableDataSet(BasicActivator.RepositoryLocator.DataExportRepository, _catalogue); - Publish(_catalogue); - } - } - else - { - var extractabilityRecord = - ((DataExportChildProvider)BasicActivator.CoreChildProvider).ExtractableDataSets.SingleOrDefault(ds => - ds.Catalogue_ID == _catalogue.ID); - if (extractabilityRecord == null) - { - Show($"{_catalogue} is already non-extractable"); - } - else - { - extractabilityRecord.DeleteInDatabase(); - Publish(_catalogue); - } - } - } -} \ No newline at end of file diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneCohortIdentificationConfiguration.cs index 0eb139ad6f..11c062d005 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneCohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneCohortIdentificationConfiguration.cs @@ -8,11 +8,14 @@ using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.DataExport.Data; using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.Providers; using Rdmp.Core.Repositories.Construction; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; +using System; +using System.Linq; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -65,6 +68,17 @@ public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) return this; } + private bool Validate(out string reason) + { + if(_cic.RootCohortAggregateContainer.GetAllAggregateConfigurationsRecursively().Select(a => a.Catalogue).Where(c => c.IsInternalDataset).Any()) + { + reason = "Configuration contains Catalogues marked as Internal"; + return false; + } + reason = null; + return true; + } + public override void Execute() { base.Execute(); @@ -74,6 +88,12 @@ public override void Execute() if (_cic == null) return; + if (!Validate(out string reason)) + { + Show("Unable to clone Cohort Identification Configuration",$"Cohort Identification Configuration is not in a valid state to clone.{reason}."); + return; + } + // Confirm creating yes/no (assuming activator is interactive) if (!_autoConfirm && BasicActivator.IsInteractive && !YesNo( "This will create a 100% copy of the entire CohortIdentificationConfiguration including all datasets, filters, parameters and set operations. Are you sure this is what you want?", diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandConvertAggregateConfigurationToPatientIndexTable.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandConvertAggregateConfigurationToPatientIndexTable.cs index ef7edb2cf8..eb9758c40a 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandConvertAggregateConfigurationToPatientIndexTable.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandConvertAggregateConfigurationToPatientIndexTable.cs @@ -40,7 +40,7 @@ public ExecuteCommandConvertAggregateConfigurationToPatientIndexTable(IBasicActi SetImpossible( $"Aggregate '{_sourceAggregateConfigurationCombineable.Aggregate}' belongs to a different Cohort Identification Configuration"); - if (cic != null && cic.ShouldBeReadOnly(out var reason)) + if (cic != null && cic.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs index c04a7929c8..acee8b5f45 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs @@ -1,5 +1,4 @@ using Rdmp.Core.Curation.Data; - namespace Rdmp.Core.CommandExecution.AtomicCommands; public class ExecuteCommandCreateDataset : BasicCommandExecution @@ -24,7 +23,7 @@ public ExecuteCommandCreateDataset(IBasicActivateItems activator, [DemandsInitia public override void Execute() { base.Execute(); - var dataset = new Curation.Data.Dataset(BasicActivator.RepositoryLocator.CatalogueRepository, _name) { DigitalObjectIdentifier = _doi, Source = _source }; + var dataset = new Rdmp.Core.Curation.Data.Datasets.Dataset(BasicActivator.RepositoryLocator.CatalogueRepository, _name) { DigitalObjectIdentifier = _doi, Source = _source }; dataset.SaveToDatabase(); _activator.Publish(dataset); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs index dc52a18a0e..a4b01172e9 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFilter.cs @@ -10,8 +10,10 @@ using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; +using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.Curation.FilterImporting; using Rdmp.Core.Curation.FilterImporting.Construction; +using Rdmp.Core.DataExport.Data; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.Repositories.Construction; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; @@ -34,44 +36,16 @@ public class ExecuteCommandCreateNewFilter : BasicCommandExecution, IAtomicComma private IFilter[] _offerFilters = []; private bool offerCatalogueFilters; - private IFilter[] _offerCohortFilters = []; - private bool offerCohortCatalogueFilters; - - public bool OfferCatalogueFilters { get => offerCatalogueFilters; set { - if (value) - { - var c = GetCatalogue(); - _offerFilters = c?.GetAllFilters(); - - if (_offerFilters == null || !_offerFilters.Any()) - SetImpossible($"There are no Filters declared in Catalogue '{c?.ToString() ?? "NULL"}'"); - } - - offerCatalogueFilters = value; } } - public bool OfferCohortCatalogueFilters - { - get => offerCohortCatalogueFilters; - set - { - var c = GetCatalogue(); - - var filters = c.CatalogueRepository.GetAllObjects().Where(af => af.GetCatalogue().ID == c.ID); - _offerCohortFilters = filters.ToArray(); - offerCohortCatalogueFilters = value; - } - } - - private ExecuteCommandCreateNewFilter(IBasicActivateItems activator) : base(activator) { Weight = DEFAULT_WEIGHT; @@ -152,7 +126,6 @@ public ExecuteCommandCreateNewFilter(IBasicActivateItems activator, IRootFilterC _factory = host.GetFilterFactory(); _container = host.RootFilterContainer; _host = host; - if (_container == null && _host is AggregateConfiguration ac) { if (ac.Catalogue.IsApiCall()) @@ -220,6 +193,7 @@ public override void Execute() container = _host.RootFilterContainer; } + // if importing an existing filter instead of creating blank if (BasedOn != null) { @@ -228,6 +202,12 @@ public override void Execute() } else if (OfferCatalogueFilters) { + + var c = GetCatalogue(); + _offerFilters = c?.GetAllFilters(); + if (_offerFilters == null || !_offerFilters.Any()) + SetImpossible($"There are no Filters declared in Catalogue '{c?.ToString() ?? "NULL"}'"); + // we want user to make decision about what to import ImportExistingFilter(container); return; @@ -257,7 +237,7 @@ private void ImportExistingFilter(IContainer container) { var wizard = new FilterImportWizard(BasicActivator); - var filters = _offerFilters.Union(_offerCohortFilters).ToArray(); + var filters = _offerFilters; var import = wizard.ImportManyFromSelection(container, filters).ToArray(); foreach (var f in import) container.AddChild(f); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs index ef4a191c58..fe08244b50 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDelete.cs @@ -50,7 +50,7 @@ public ExecuteCommandDelete(IBasicActivateItems activator, SetImpossible("Cannot delete root containers"); var reason = ""; - if (_deletables.Any(d => d is IMightBeReadOnly ro && ro.ShouldBeReadOnly(out reason))) + if (_deletables.Any(d => d is IMightBeReadOnly ro && ro.ShouldBeReadOnly(this.GetType().Name, out reason))) SetImpossible(reason); Weight = 50.4f; @@ -80,18 +80,9 @@ private string GetDeleteVerbIfAny() public override void Execute() { base.Execute(); - - // if the thing we are deleting is important and sensitive then we should use a transaction - if (_deletables.Count > 1 || ShouldUseTransactionsWhenDeleting(_deletables.FirstOrDefault())) - { - ExecuteWithCommit(ExecuteImpl, GetDescription(), - _deletables.OfType().ToArray()); - PublishNearest(); - } - else - { - ExecuteImpl(); - } + ExecuteWithCommit(ExecuteImpl, GetDescription(), + _deletables.OfType().ToArray()); + PublishNearest(); } private static bool ShouldUseTransactionsWhenDeleting(IDeleteable deletable) => diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs index 734a81c431..8f65b79385 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs @@ -4,9 +4,9 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public sealed class ExecuteCommandDeleteDataset: BasicCommandExecution { - private readonly Curation.Data.Dataset _dataset; + private readonly Curation.Data.Datasets.Dataset _dataset; private readonly IBasicActivateItems _activator; -public ExecuteCommandDeleteDataset(IBasicActivateItems activator, [DemandsInitialization("The Dataset to delete")]Curation.Data.Dataset dataset) +public ExecuteCommandDeleteDataset(IBasicActivateItems activator, [DemandsInitialization("The Dataset to delete")]Curation.Data.Datasets.Dataset dataset) { _dataset = dataset; _activator = activator; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs index d2008fff56..cb94ded66b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDisableOrEnable.cs @@ -62,7 +62,7 @@ private void UpdateViabilityForTarget(IDisableable target) } } - if (target is IMightBeReadOnly ro && ro.ShouldBeReadOnly(out var reason)) + if (target is IMightBeReadOnly ro && ro.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs new file mode 100644 index 0000000000..443961c50a --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToConfluence.cs @@ -0,0 +1,286 @@ +using Newtonsoft.Json; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Dataset.Confluence; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands +{ + /// + /// Uses provided Confluence credentials to populate a tree of pages about the public catalogues in RDMP + /// + public class ExecuteCommandExportCataloguesToConfluence : BasicCommandExecution, IAtomicCommand + { + + private readonly IBasicActivateItems _activator; + private readonly string _subdomain; + private readonly int _spaceId; + private readonly string _apiKey; + private readonly string _owner; + private readonly string _description; + private readonly bool _isServiceAccount; + private readonly int? _rootPageParent; + private readonly HttpClient _client = new(); + private readonly Dictionary _cataloguePageLookups = []; + private string rootParentId = null; + private int rootParentVersion = 0; + private class ConfluencePageBody + { + public string representation { get; set; } = "storage"; + public string value { get; set; } + } + + private class ConfluencePageVersion + { + public int number { get; set; } + } + + private class ConfluencePagePostRequest + { + public string spaceId { get; set; } + public string status { get; set; } = "current"; + public string title { get; set; } = "Catalogues"; + public string parentId { get; set; } + public ConfluencePageBody body { get; set; } + + } + + private class ConfluencePagePutRequest + { + public int id { get; set; } + public string status { get; set; } = "current"; + public string title { get; set; } = "Catalogues"; + public ConfluencePageBody body { get; set; } + public ConfluencePageVersion version { get; set; } + + } + + private class ConfluenceLinks + { + public string webui { get; set; } + } + + private class ConfluencePostError + { + public int status { get; set; } + public string code { get; set; } + public string title { get; set; } + public string detail { get; set; } + } + private class ConfluencePostErrors + { + public List errors { get; set; } + } + + private class ConfluenceGetResults + { + public List results { get; set; } + } + private class ConfluenceGetResult + { + public ConfluencePageVersion version { get; set; } + public string id { get; set; } + } + + private class ConfluencePageResponse + { + public string id { get; set; } + public ConfluenceLinks _links { get; set; } + public ConfluencePageVersion version { get; set; } + } + + public ExecuteCommandExportCataloguesToConfluence(IBasicActivateItems activator, string subdomain, int spaceId, string apiKey, string owner, string description, bool isServiceaccount, int? rootPageParent = null) + { + _activator = activator; + _subdomain = subdomain; + _spaceId = spaceId; + _apiKey = apiKey; + _owner = owner; + _description = description; + _isServiceAccount = isServiceaccount; + _rootPageParent = rootPageParent; + } + + private static ConfluencePageResponse ResponseToConfluenceResponseObject(HttpResponseMessage response) + { + var content = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + return JsonConvert.DeserializeObject(content); + } + + private static ConfluencePostErrors PostResponseToConfluenceErrorsObject(HttpResponseMessage response) + { + var content = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + return JsonConvert.DeserializeObject(content); + } + + private ConfluenceGetResults GetConfluencePage(string uri, string title) + { + var result = Task.Run(async () => await _client.GetAsync($"{uri}?title={title}&space-id={_spaceId}")).Result; + var content = Task.Run(async () => await result.Content.ReadAsStringAsync()).Result; + return JsonConvert.DeserializeObject(content); + } + + + private void UpdateContainerPage(ConfluencePageBuilder builder, string uri) + { + var rootPageHTML = builder.BuildContainerPage(_cataloguePageLookups); + var putRequest = new ConfluencePagePutRequest() + { + id = int.Parse(rootParentId), + title = $"{_owner} Catalogues", + status = "current", + body = new ConfluencePageBody() { value = rootPageHTML }, + version = new ConfluencePageVersion() + { + number = rootParentVersion + 1 + } + }; + var response = Task.Run(async () => await _client.PutAsJsonAsync($"{uri}/{rootParentId}", putRequest)).Result; + if (response.StatusCode == System.Net.HttpStatusCode.OK) + { + var putResponseContent = ResponseToConfluenceResponseObject(response); + rootParentId = putResponseContent.id; + rootParentVersion = putResponseContent.version.number; + } + else + { + throw new Exception("Unable to update container page"); + } + } + + private bool PageAlreadyExists(ConfluencePostErrors errors) + { + return errors.errors.Count == 1 && errors.errors[0].title.Contains("page with this title already exists"); + } + + private void CreateContainerPage(ConfluencePageBuilder builder, string uri) + { + var rootPageHTML = builder.BuildContainerPage(_cataloguePageLookups); + var request = new ConfluencePagePostRequest() + { + spaceId = _spaceId.ToString(), + title = $"{_owner} Catalogues", + body = new ConfluencePageBody() { value = rootPageHTML } + }; + if (_rootPageParent != null) + { + request.parentId = _rootPageParent.ToString() ; + } + HttpResponseMessage response = Task.Run(async () => await _client.PostAsJsonAsync(uri, request)).Result; + if (response.StatusCode == System.Net.HttpStatusCode.OK) + { + var confluencePostResponseObject = ResponseToConfluenceResponseObject(response); + rootParentId = confluencePostResponseObject.id; + rootParentVersion = confluencePostResponseObject.version.number; + } + else if (response.StatusCode == System.Net.HttpStatusCode.BadRequest) + { + var confluenceErrorsResponse = PostResponseToConfluenceErrorsObject(response); + if (PageAlreadyExists(confluenceErrorsResponse)) + { + var foundPages = GetConfluencePage(uri, request.title); + if (foundPages.results.Count != 1) + { + throw new Exception($"Multiple pages in pace {_spaceId} with name {request.title}"); + } + rootParentId = foundPages.results[0].id; + rootParentVersion = foundPages.results[0].version.number; + UpdateContainerPage(builder, uri); + } + else + { + throw new Exception($"Unexpected Error when creating container page - {confluenceErrorsResponse.errors[0].title}"); + } + } + else + { + throw new Exception($"Unable to create root page - {response.StatusCode}"); + } + } + + private void CreateCataloguePage(Catalogue catalogue, ConfluencePageBuilder builder, string uri) + { + var cataloguePageHTML = builder.BuildHTMLForCatalogue(catalogue); + var request = new ConfluencePagePostRequest() + { + spaceId = _spaceId.ToString(), + title = $"{catalogue.Name}{(catalogue.Acronym != null ? $" ({catalogue.Acronym})" : "")}", + parentId = rootParentId, + body = new ConfluencePageBody() { value = cataloguePageHTML } + }; + var response = Task.Run(async () => await _client.PostAsJsonAsync(uri, request)).Result; + if (response.StatusCode == System.Net.HttpStatusCode.OK) + { + var responseContent = ResponseToConfluenceResponseObject(response); + _cataloguePageLookups.TryAdd(catalogue.ID, responseContent._links.webui); + } + else if (response.StatusCode == System.Net.HttpStatusCode.BadRequest) + { + var responseContent = PostResponseToConfluenceErrorsObject(response); + if (PageAlreadyExists(responseContent)) + { + var getResults = GetConfluencePage(uri, request.title); + if (getResults.results.Count != 1) + { + throw new Exception($"Multiple pages in pace {_spaceId} with name {request.title}"); + } + var id = getResults.results[0].id; + var version = getResults.results[0].version.number; + + var cataloguePutRequest = new ConfluencePagePutRequest() + { + id = int.Parse(id), + title = request.title, + status = "current", + body = new ConfluencePageBody() { value = cataloguePageHTML }, + version = new ConfluencePageVersion() + { + number = version + 1 + } + }; + response = Task.Run(async () => await _client.PutAsJsonAsync($"{uri}/{id}", cataloguePutRequest)).Result; + if (response.StatusCode != System.Net.HttpStatusCode.OK) + { + throw new Exception($"Unable to PUT update for catalogue '{catalogue.Name}' - {response.StatusCode}"); + } + } + } + else + { + throw new Exception($"Unknown status code when working with catalogue '{catalogue.Name}' - {response.StatusCode}"); + } + } + + public override void Execute() + { + base.Execute(); + _client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", _apiKey); + + var catalogues = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects() + .Where(c => !c.IsDeprecated && !c.IsInternalDataset && !c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) + .ToList(); + var builder = new ConfluencePageBuilder(catalogues, _owner, _description, _subdomain); + var uri = _isServiceAccount ? $"https://api.atlassian.com/ex/confluence/{_subdomain}/api/v2/pages" : $"https://{_subdomain}.atlassian.net/wiki/api/v2/pages"; + CreateContainerPage(builder, uri); + if (rootParentId != null) + { + foreach (var catalogue in catalogues) + { + CreateCataloguePage(catalogue, builder, uri); + } + UpdateContainerPage(builder, uri);//re-add the links for the catalogue pages to the home page + + } + else + { + throw new Exception($"Unable to find root id in response"); + + } + } + + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportCohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportCohortIdentificationConfiguration.cs index 45d145e372..cbeac6b2d8 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportCohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportCohortIdentificationConfiguration.cs @@ -36,7 +36,7 @@ public ExecuteCommandImportCohortIdentificationConfiguration(IBasicActivateItems return; } - if (intoContainer.ShouldBeReadOnly(out var reason)) SetImpossible(reason); + if (intoContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); // if we don't know what to import yet then we should have the // 'more choices to come' suffix diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs new file mode 100644 index 0000000000..e00b815d0d --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -0,0 +1,70 @@ +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NPOI.OpenXmlFormats.Vml.Office; + +namespace Rdmp.Core.CommandExecution.AtomicCommands +{ + public class ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider : BasicCommandExecution, IAtomicCommand + { + + private readonly IBasicActivateItems _activator; + private readonly IDatasetProvider _provider; + private readonly bool _includeExtractable; + private readonly bool _includeInternal; + private readonly bool _includeProjectSpecific; + private readonly bool _includeDeprecated; + private readonly bool _autoUpdate; + + public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicActivateItems activator, IDatasetProvider provider, bool includeExtractable, bool includeInternal, bool includeProjectSpecific, bool includeDeprecated, bool autoUpdate) + { + _activator = activator; + _provider = provider; + _includeExtractable = includeExtractable; + _includeInternal = includeInternal; + _includeProjectSpecific = includeProjectSpecific; + _includeDeprecated = includeDeprecated; + _autoUpdate = autoUpdate; + } + + public List GetCatalogues() + { + var catalogues = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); + if (!_includeInternal) + { + catalogues = catalogues.Where(c => !c.IsInternalDataset).ToList(); + } + if (!_includeProjectSpecific) + { + catalogues = catalogues.Where(c => !c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)).ToList(); + } + if (!_includeDeprecated) + { + catalogues = catalogues.Where(c => !c.IsDeprecated).ToList(); + } + if (!_includeExtractable) + { + catalogues = catalogues.Where(c => !c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList(); + } + //todo what to do about non-extractable catalogues???? + return catalogues; + } + + + public override void Execute() + { + var catalogues = GetCatalogues(); + foreach (var catalogue in catalogues) + { + var dataset = _provider.Create(catalogue); + var ds = _provider.AddExistingDatasetWithReturn(null, dataset.GetID()); + var cmd = new ExecuteCommandLinkCatalogueToDataset(_activator, catalogue, ds,_autoUpdate); + cmd.Execute(); + } + } + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs index 683d910a14..69d8d3b945 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs @@ -7,19 +7,20 @@ using System; using Rdmp.Core.Curation.Data; using System.Linq; +using Rdmp.Core.Curation.Data.Datasets; namespace Rdmp.Core.CommandExecution.AtomicCommands; public sealed class ExecuteCommandLinkCatalogueToDataset : BasicCommandExecution { private readonly Catalogue _catalogue; - private readonly Curation.Data.Dataset _dataset; - private readonly bool _linkAll; - public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [DemandsInitialization("The catalogue To link")]Catalogue catalogue, [DemandsInitialization("The dataset to link to")]Curation.Data.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) + private readonly Curation.Data.Datasets.Dataset _dataset; + private readonly bool _autoUpdate; + public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [DemandsInitialization("The catalogue To link")]Catalogue catalogue, [DemandsInitialization("The dataset to link to")]Curation.Data.Datasets.Dataset dataset, bool autoUpdate = true) : base(activator) { _catalogue = catalogue; _dataset = dataset; - _linkAll = linkAllOtherColumns; + _autoUpdate = autoUpdate; if (_catalogue is null) SetImpossible("No Catalogue Selected"); if (_dataset is null) SetImpossible("No Dataset Selected"); @@ -29,21 +30,9 @@ public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [Dema public override void Execute() { base.Execute(); - var items = _catalogue.CatalogueItems.ToList(); - foreach (var ci in items.Select(static item => item.ColumnInfo).Where(ci => ci?.Dataset_ID != _dataset.ID)) - { - ci.Dataset_ID = _dataset.ID; - ci.SaveToDatabase(); - if (!_linkAll) continue; - - var databaseName = ci.Name[..ci.Name.LastIndexOf('.')]; - var catalogueItems = ci.CatalogueRepository.GetAllObjects().Where(ci => ci.Name[..ci.Name.LastIndexOf(".", StringComparison.Ordinal)] == databaseName).ToList(); - foreach (var aci in catalogueItems) - { - aci.Dataset_ID = _dataset.ID; - aci.SaveToDatabase(); - } - } + var linkage = new CatalogueDatasetLinkage(_catalogue.CatalogueRepository, _catalogue, _dataset,_autoUpdate); + linkage.SaveToDatabase(); + Publish(linkage); } } \ No newline at end of file diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs index 7eac784215..57034e3851 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs @@ -13,9 +13,9 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public sealed class ExecuteCommandLinkColumnInfoToDataset : BasicCommandExecution { private readonly ColumnInfo _columnInfo; - private readonly Curation.Data.Dataset _dataset; + private readonly Curation.Data.Datasets.Dataset _dataset; private readonly bool _linkAll; - public ExecuteCommandLinkColumnInfoToDataset(IBasicActivateItems activator, [DemandsInitialization("The column to link")] ColumnInfo columnInfo, [DemandsInitialization("The dataset to link to")] Curation.Data.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) + public ExecuteCommandLinkColumnInfoToDataset(IBasicActivateItems activator, [DemandsInitialization("The column to link")] ColumnInfo columnInfo, [DemandsInitialization("The dataset to link to")] Curation.Data.Datasets.Dataset dataset, bool linkAllOtherColumns = true) : base(activator) { _columnInfo = columnInfo; _dataset = dataset; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueInternal.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueInternal.cs new file mode 100644 index 0000000000..905bb0e768 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueInternal.cs @@ -0,0 +1,77 @@ +// Copyright (c) The University of Dundee 2018-2025 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Repositories.Construction; +using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +public class ExecuteCommandMakeCatalogueInternal : BasicCommandExecution, IAtomicCommandWithTarget +{ + private ICatalogue _catalogue; + + [UseWithObjectConstructor] + public ExecuteCommandMakeCatalogueInternal(IBasicActivateItems itemActivator, ICatalogue catalogue) : this(itemActivator) + { + SetCatalogue(catalogue); + } + + public ExecuteCommandMakeCatalogueInternal(IBasicActivateItems itemActivator) : base(itemActivator) + { + UseTripleDotSuffix = true; + } + + public override string GetCommandHelp() => + "Mark the Catalogue as Internal to restrict its extractability"; + + public override void Execute() + { + if (_catalogue == null) + SetCatalogue(SelectOne(BasicActivator.RepositoryLocator.CatalogueRepository)); + + if (_catalogue == null) + return; + + _catalogue.IsInternalDataset = true; + _catalogue.SaveToDatabase(); + Publish(_catalogue); + } + + public override Image GetImage(IIconProvider iconProvider) => BasicActivator.CoreIconProvider.GetImage(_catalogue, OverlayKind.Internal); + + public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) + { + switch (target) + { + case Catalogue catalogue: + SetCatalogue(catalogue); + break; + default: + break; + } + + return this; + } + + private void SetCatalogue(ICatalogue catalogue) + { + ResetImpossibleness(); + + _catalogue = catalogue; + + if (catalogue == null) + { + SetImpossible("Catalogue cannot be null"); + return; + } + + if (_catalogue.IsInternalDataset) + SetImpossible("Catalogue is already marked as Internal"); + } +} \ No newline at end of file diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueNotInternal.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueNotInternal.cs new file mode 100644 index 0000000000..81d27a05fc --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueNotInternal.cs @@ -0,0 +1,79 @@ +// Copyright (c) The University of Dundee 2018-2025 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.Repositories.Construction; +using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +public class ExecuteCommandMakeCatalogueNotInternal : BasicCommandExecution, IAtomicCommandWithTarget +{ + private ICatalogue _catalogue; + + [UseWithObjectConstructor] + public ExecuteCommandMakeCatalogueNotInternal(IBasicActivateItems itemActivator, ICatalogue catalogue) : this(itemActivator) + { + SetCatalogue(catalogue); + } + + public ExecuteCommandMakeCatalogueNotInternal(IBasicActivateItems itemActivator) : base(itemActivator) + { + UseTripleDotSuffix = true; + } + + public override string GetCommandHelp() => + "Mark the Catalogue as not Internal"; + + public override void Execute() + { + if (_catalogue == null) + SetCatalogue(SelectOne(BasicActivator.RepositoryLocator.CatalogueRepository)); + + if (_catalogue == null) + return; + + _catalogue.IsInternalDataset = false; + _catalogue.SaveToDatabase(); + Publish(_catalogue); + } + + public override Image GetImage(IIconProvider iconProvider) => + Image.Load(CatalogueIcons.Catalogue); + + public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) + { + switch (target) + { + case Catalogue catalogue: + SetCatalogue(catalogue); + break; + default: + break; + } + + return this; + } + + private void SetCatalogue(ICatalogue catalogue) + { + ResetImpossibleness(); + + _catalogue = catalogue; + + if (catalogue == null) + { + SetImpossible("Catalogue cannot be null"); + return; + } + + if (!_catalogue.IsInternalDataset) + SetImpossible("Catalogue is not marked as Internal"); + } +} \ No newline at end of file diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs index 8039dbaeae..5a6571843e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs @@ -4,16 +4,18 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Linq; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.Icons.IconProvision; +using Rdmp.Core.Providers; using Rdmp.Core.Repositories.Construction; +using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; +using System; +using System.Collections.Generic; +using System.Linq; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -23,19 +25,23 @@ public class ExecuteCommandMakeCatalogueProjectSpecific : BasicCommandExecution, private IProject _project; private List _existingProjectIDs; private readonly bool _force = false; + private bool _hasRanCatalogueValidation = false; + private readonly IBasicActivateItems _activator; [UseWithObjectConstructor] public ExecuteCommandMakeCatalogueProjectSpecific(IBasicActivateItems itemActivator, ICatalogue catalogue, - IProject project, [DemandsInitialization("Ignore Validation",DemandType.Unspecified,defaultValue:false)]bool force) : this(itemActivator) + IProject project, [DemandsInitialization("Ignore Validation", DemandType.Unspecified, defaultValue: false)] bool force) : this(itemActivator) { - SetCatalogue(catalogue); + _catalogue = catalogue; _project = project; _force = force; + _activator = itemActivator; } public ExecuteCommandMakeCatalogueProjectSpecific(IBasicActivateItems itemActivator) : base(itemActivator) { UseTripleDotSuffix = true; + _activator = itemActivator; } public override string GetCommandHelp() => @@ -44,9 +50,28 @@ public override string GetCommandHelp() => public override void Execute() { if (_catalogue == null) - SetCatalogue(SelectOne(BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList())); - GetExistingProjectIDs(); + { + var catalogues = BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); + if (!catalogues.Any()) + { + Show($"No valid catalogues found to make project specific."); + return; + } + SetCatalogue(SelectOne(catalogues)); + } + if (!_hasRanCatalogueValidation) + { + SetCatalogue(_catalogue); + } + if (_existingProjectIDs is null) + GetExistingProjectIDs(); + var projects = GetListOfValidProjects(); + if (!projects.Any()) + { + Show($"No valid projects found to make {_catalogue.Name} project specific."); + return; + } _project ??= SelectOne(GetListOfValidProjects()); if (_project == null || _catalogue == null) @@ -79,20 +104,24 @@ public IAtomicCommandWithTarget SetTarget(DatabaseEntity target) private List GetListOfValidProjects() { - var availableProjects = BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(p => !p.GetAllProjectCatalogues().Contains(_catalogue)); - return availableProjects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(BasicActivator.RepositoryLocator.DataExportRepository, _catalogue, p, _existingProjectIDs)).ToList(); + var dataExportChildProvider = ((DataExportChildProvider)_activator.CoreChildProvider); + var eds = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWithParent(_catalogue); + var edsp = _activator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(edsp => eds.Contains(edsp.DataSet)); + var pti = edsp.Select(e => e.Project_ID).ToList(); + var validProjects = dataExportChildProvider.Projects.Where(p => _force ||(!pti.Contains(p.ID) && ProjectSpecificCatalogueManager.CanMakeCatalogueProjectSpecific(_activator.RepositoryLocator.DataExportRepository, _catalogue, p, pti))); + return validProjects.ToList(); } private void GetExistingProjectIDs() { - var existingProjects = BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(p => p.GetAllProjectCatalogues().Contains(_catalogue)); + var dataExportChildProvider = ((DataExportChildProvider)_activator.CoreChildProvider); + var existingProjects = dataExportChildProvider.Projects.Where(p => dataExportChildProvider.ExtractableDataSetProjects.Where(edsp => edsp.Project_ID == p.ID).Select(edsp => edsp.DataSet.Catalogue).Contains(_catalogue)); _existingProjectIDs = existingProjects.Select(p => p.ID).ToList(); } private void SetCatalogue(ICatalogue catalogue) { ResetImpossibleness(); - _catalogue = catalogue; GetExistingProjectIDs(); if (catalogue == null) @@ -100,9 +129,7 @@ private void SetCatalogue(ICatalogue catalogue) SetImpossible("Catalogue cannot be null"); return; } - var status = _catalogue.GetExtractabilityStatus(BasicActivator.RepositoryLocator.DataExportRepository); - if (!GetListOfValidProjects().Any() && !_force) { SetImpossible("No valid Projects available"); @@ -111,6 +138,7 @@ private void SetCatalogue(ICatalogue catalogue) if (!status.IsExtractable) SetImpossible("Catalogue must first be made Extractable"); + var ei = _catalogue.GetAllExtractionInformation(ExtractionCategory.Any); if (!ei.Any()) SetImpossible("Catalogue has no extractable columns"); @@ -118,5 +146,6 @@ private void SetCatalogue(ICatalogue catalogue) if (ei.Count(e => e.IsExtractionIdentifier) < 1) SetImpossible("Catalogue must have at least 1 IsExtractionIdentifier column"); + _hasRanCatalogueValidation = true; } } \ No newline at end of file diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakePatientIndexTableIntoRegularCohortIdentificationSetAgain.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakePatientIndexTableIntoRegularCohortIdentificationSetAgain.cs index a2431b0556..7323fd4bfe 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakePatientIndexTableIntoRegularCohortIdentificationSetAgain.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakePatientIndexTableIntoRegularCohortIdentificationSetAgain.cs @@ -39,7 +39,7 @@ public ExecuteCommandMakePatientIndexTableIntoRegularCohortIdentificationSetAgai SetImpossible( $"The following Cohort Set(s) use this PatientIndex table:{string.Join(",", _sourceAggregateCommand.JoinableUsersIfAny.Select(j => j.ToString()))}"); - if (_targetCohortAggregateContainer.ShouldBeReadOnly(out var reason)) + if (_targetCohortAggregateContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs index a8a0511c3f..8b6bf82bbb 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs @@ -19,7 +19,7 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; public class ExecuteCommandMakeProjectSpecificCatalogueNormalAgain : BasicCommandExecution, IAtomicCommand { private Catalogue _catalogue; - private readonly List _extractableDataSets; + private List _extractableDataSets; private ExtractableDataSet _extractableDataSet; private Project _selectedProj; @@ -36,28 +36,29 @@ public ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(IBasicActivateItems SetImpossible("Data Export functionality is not available"); return; } + } + + public override string GetCommandHelp() => + "Take a dataset that was previously only usable with extractions of a specific project and make it free for use in any extraction project"; + + public override void Execute() + { + var dataExportRepository = BasicActivator.RepositoryLocator.DataExportRepository; - _extractableDataSets = dataExportRepository.GetAllObjectsWithParent(catalogue).Where(eds => eds.Projects.Any() && eds.Projects.Select(p =>ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(dataExportRepository, catalogue, eds,p)).Contains(true)).ToList(); + base.Execute(); + _extractableDataSets = dataExportRepository.GetAllObjectsWithParent(_catalogue).Where(eds => eds.Projects.Any() && eds.Projects.Select(p => ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(dataExportRepository, _catalogue, eds, p)).Contains(true)).ToList(); if (!_extractableDataSets.Any()) { SetImpossible("Cannot make Catalogue Non-Project specific"); return; } - - } - public override string GetCommandHelp() => - "Take a dataset that was previously only usable with extractions of a specific project and make it free for use in any extraction project"; - public override void Execute() - { - base.Execute(); if (_extractableDataSet is null) { - var dataExportRepository = BasicActivator.RepositoryLocator.DataExportRepository; - var projectIds = _extractableDataSets.SelectMany(eds => eds.Projects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(dataExportRepository,_catalogue,eds,p))).Select(p => p.ID); + var projectIds = _extractableDataSets.SelectMany(eds => eds.Projects.Where(p => ProjectSpecificCatalogueManager.CanMakeCatalogueNonProjectSpecific(dataExportRepository,_catalogue, eds,p))).Select(p => p.ID); _selectedProj = SelectOne(BasicActivator.RepositoryLocator.DataExportRepository.GetAllObjectsInIDList(projectIds).ToList()); if (_selectedProj is null) return; _extractableDataSet = _extractableDataSets.FirstOrDefault(eds => eds.Projects.Select(p => p.ID).Contains(_selectedProj.ID)); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveAggregateIntoContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveAggregateIntoContainer.cs index 34dd194502..cc4949587a 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveAggregateIntoContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveAggregateIntoContainer.cs @@ -39,7 +39,7 @@ public ExecuteCommandMoveAggregateIntoContainer(IBasicActivateItems activator, _sourceAggregateCommand.ContainerIfAny.Equals(targetCohortAggregateContainer)) SetImpossible("Aggregate is already in container"); - if (targetCohortAggregateContainer.ShouldBeReadOnly(out var reason)) + if (targetCohortAggregateContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveCohortAggregateContainerIntoSubContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveCohortAggregateContainerIntoSubContainer.cs index 7d122d6c98..f82edf3214 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveCohortAggregateContainerIntoSubContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveCohortAggregateContainerIntoSubContainer.cs @@ -35,7 +35,7 @@ public ExecuteCommandMoveCohortAggregateContainerIntoSubContainer(IBasicActivate if (_sourceCohortAggregateContainer.AggregateContainer.Equals(_targetCohortAggregateContainer)) SetImpossible("Cannot move a container into itself"); - if (_targetCohortAggregateContainer.ShouldBeReadOnly(out var reason)) + if (_targetCohortAggregateContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveContainerIntoContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveContainerIntoContainer.cs index f1d4de1e4c..8b85311d2b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveContainerIntoContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveContainerIntoContainer.cs @@ -30,7 +30,7 @@ public ExecuteCommandMoveContainerIntoContainer(IBasicActivateItems activator, if (containerCombineable.AllSubContainersRecursive.Contains(targetContainer)) SetImpossible("You cannot move a container (AND/OR) into one of its own subcontainers"); - if (targetContainer.ShouldBeReadOnly(out var reason)) + if (targetContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveFilterIntoContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveFilterIntoContainer.cs index 615a96433b..556f8abfa5 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveFilterIntoContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMoveFilterIntoContainer.cs @@ -30,7 +30,7 @@ public ExecuteCommandMoveFilterIntoContainer(IBasicActivateItems activator, Filt if (!filterCombineable.AllContainersInEntireTreeFromRootDown.Contains(targetContainer)) SetImpossible("Filters can only be moved within their own container tree"); - if (targetContainer.ShouldBeReadOnly(out var reason)) + if (targetContainer.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRename.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRename.cs index 1ea4106247..1c5b391206 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRename.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRename.cs @@ -27,7 +27,7 @@ public ExecuteCommandRename(IBasicActivateItems activator, INamed nameable) : ba case ITableInfo: SetImpossible("TableInfos cannot not be renamed"); break; - case IMightBeReadOnly ro when ro.ShouldBeReadOnly(out var reason): + case IMightBeReadOnly ro when ro.ShouldBeReadOnly(this.GetType().Name, out var reason): SetImpossible(reason); break; } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs index 5599c403c9..6ad42f26ae 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetColumnSettingBase.cs @@ -4,10 +4,11 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Linq; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; +using Rdmp.Core.Providers; +using System; +using System.Linq; namespace Rdmp.Core.CommandExecution.AtomicCommands; @@ -49,6 +50,7 @@ public ExecuteCommandSetColumnSettingBase( _catalogue.ClearAllInjections(); _commandProperty = commandProperty; + if (inConfiguration != null) { SetImpossibleIfReadonly(_inConfiguration); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetContainerOperation.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetContainerOperation.cs index bb8818160a..f141522f7b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetContainerOperation.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetContainerOperation.cs @@ -29,7 +29,7 @@ public ExecuteCommandSetContainerOperation(IBasicActivateItems activator, Cohort _container = container; _operation = operation; - if (container.ShouldBeReadOnly(out var reason)) SetImpossible(reason); + if (container.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); Weight = _operation switch { diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs index c838effd4b..67816f169c 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandSetQueryCachingDatabase.cs @@ -24,8 +24,7 @@ public ExecuteCommandSetQueryCachingDatabase(IBasicActivateItems activator, Coho base(activator) { _cic = cic; - _caches = BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(static s => s.WasCreatedBy(new QueryCachingPatcher())).ToArray(); + _caches = activator.CoreChildProvider.AllExternalServers.Where(es => es.WasCreatedBy(new QueryCachingPatcher())).ToArray(); if (!_caches.Any()) SetImpossible("There are no Query Caching databases set up"); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecutecommandAddCohortSubContainer.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecutecommandAddCohortSubContainer.cs index 01725cb730..7e1d61675b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecutecommandAddCohortSubContainer.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecutecommandAddCohortSubContainer.cs @@ -22,7 +22,7 @@ public ExecuteCommandAddCohortSubContainer(IBasicActivateItems activator, Cohort Weight = 0.12f; _container = container; - if (container.ShouldBeReadOnly(out var reason)) SetImpossible(reason); + if (container.ShouldBeReadOnly(this.GetType().Name, out var reason)) SetImpossible(reason); } public override Image GetImage(IIconProvider iconProvider) => diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/Sharing/ExecuteCommandExportInDublinCoreFormat.cs b/Rdmp.Core/CommandExecution/AtomicCommands/Sharing/ExecuteCommandExportInDublinCoreFormat.cs index ab5bbcde5c..1a7bbaca08 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/Sharing/ExecuteCommandExportInDublinCoreFormat.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/Sharing/ExecuteCommandExportInDublinCoreFormat.cs @@ -12,19 +12,19 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands.Sharing; public class ExecuteCommandExportInDublinCoreFormat : BasicCommandExecution, IAtomicCommand { - private readonly DublinCoreDefinition _definition; + private readonly Catalogue _catalogue; private FileInfo _toExport; - private readonly DublinCoreTranslater _translater = new(); public ExecuteCommandExportInDublinCoreFormat(IBasicActivateItems activator, Catalogue catalogue) : base(activator) { - _definition = DublinCoreTranslater.GenerateFrom(catalogue); - UseTripleDotSuffix = true; + _catalogue = catalogue; + UseTripleDotSuffix = true; } public override void Execute() { base.Execute(); + DublinCoreDefinition _definition = DublinCoreTranslater.GenerateFrom(_catalogue); if ((_toExport ??= BasicActivator.SelectFile("Dublin Core Xml|*.xml")) == null) return; diff --git a/Rdmp.Core/CommandExecution/BasicActivateItems.cs b/Rdmp.Core/CommandExecution/BasicActivateItems.cs index 5f1b8cf779..e0583256f6 100644 --- a/Rdmp.Core/CommandExecution/BasicActivateItems.cs +++ b/Rdmp.Core/CommandExecution/BasicActivateItems.cs @@ -380,21 +380,20 @@ protected virtual bool InteractiveDelete(IDeleteable deletable) { case Catalogue c: { - if (c.GetExtractabilityStatus(RepositoryLocator.DataExportRepository).IsExtractable) + var extractableDataSets = RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", c.ID); + if (extractableDataSets.Any()) { - if (YesNo( - "Catalogue must first be made non extractable before it can be deleted, mark non extractable?", - "Make Non Extractable")) + foreach (var ds in extractableDataSets) { - var cmd = new ExecuteCommandChangeExtractability(this, c); - cmd.Execute(); - } - else - { - return false; + var selectedDatasets = RepositoryLocator.DataExportRepository.GetAllObjectsWhere("ExtractableDataSet_ID", ds.ID); + if (selectedDatasets.Any()) + { + this.Show("Catalogue is used in a number of projects. Remove this catalogue from all projects to allow deletion"); + return false; + } } + foreach (var ds in extractableDataSets) ds.DeleteInDatabase(); } - break; } case ExtractionFilter f: @@ -667,6 +666,12 @@ public virtual CohortCreationRequest GetCohortCreationRequest(ExternalCohortTabl RepositoryLocator.DataExportRepository, cohortInitialDescription); } + public virtual IProject CohortCommitProjectSelect(IProject currentProject, Project[] projects) + { + return currentProject; + } + + /// public virtual CohortHoldoutLookupRequest GetCohortHoldoutLookupRequest(ExternalCohortTable externalCohortTable, IProject project, CohortIdentificationConfiguration cic) { @@ -674,7 +679,7 @@ public virtual CohortHoldoutLookupRequest GetCohortHoldoutLookupRequest(External if (!TypeText("Name", "Enter name for cohort", 255, null, out var name, false)) throw new Exception("User chose not to enter a name for the cohort and none was provided"); - return new CohortHoldoutLookupRequest(cic, "empty", 1,false,"",""); + return new CohortHoldoutLookupRequest(cic, "empty", 1, false, "", ""); } /// diff --git a/Rdmp.Core/CommandExecution/BasicCommandExecution.cs b/Rdmp.Core/CommandExecution/BasicCommandExecution.cs index 07de059b51..f7d90f428f 100644 --- a/Rdmp.Core/CommandExecution/BasicCommandExecution.cs +++ b/Rdmp.Core/CommandExecution/BasicCommandExecution.cs @@ -86,7 +86,7 @@ public abstract class BasicCommandExecution : IAtomicCommand protected void SetImpossibleIfReadonly(IMightBeReadOnly m) { - if (m?.ShouldBeReadOnly(out var reason) == true) + if (m?.ShouldBeReadOnly(this.GetType().Name,out var reason) == true) SetImpossible( $"{(m is IContainer ? "Container" : '\'' + m.ToString() + '\'')} is readonly because:{reason}"); } diff --git a/Rdmp.Core/CommandExecution/GoToCommandFactory.cs b/Rdmp.Core/CommandExecution/GoToCommandFactory.cs index 1d5b8b0d2d..4852d7589c 100644 --- a/Rdmp.Core/CommandExecution/GoToCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/GoToCommandFactory.cs @@ -47,8 +47,7 @@ public IEnumerable GetCommands(object forObject) if (Is(forObject, out IMapsDirectlyToDatabaseTable mt)) { // Go to import / export definitions - var export = _activator.RepositoryLocator.CatalogueRepository.GetReferencesTo(mt) - .FirstOrDefault(); + var export = _activator.CoreChildProvider.AllExports.FirstOrDefault(export => export.IsReferenceTo(mt)); if (export != null) yield return new ExecuteCommandShow(_activator, export, 0, true) @@ -188,14 +187,11 @@ public IEnumerable GetCommands(object forObject) if (Is(forObject, out ExtractionFilter masterFilter)) { yield return new ExecuteCommandShow(_activator, () => - _activator.RepositoryLocator.CatalogueRepository - .GetAllObjectsWhere("ClonedFromExtractionFilter_ID", masterFilter.ID) - .Select(f => f.GetAggregate()) + _activator.CoreChildProvider.AllAggregateFilters.Where(af => af.ClonedFromExtractionFilter_ID == masterFilter.ID).Select(f => f.GetAggregate()) .Where(a => a != null).Distinct() ) { OverrideCommandName = "Usages (in Cohort Builder)" }; - yield return new ExecuteCommandShow(_activator, () => _activator.RepositoryLocator.DataExportRepository .GetAllObjectsWhere("ClonedFromExtractionFilter_ID", masterFilter.ID) @@ -264,12 +260,14 @@ public IEnumerable GetCommands(object forObject) if (Is(forObject, out Catalogue catalogue)) { - foreach (var lmd in catalogue.LoadMetadatas()) + var lmdLinkage = _activator.CoreChildProvider.AllLoadMetadataCatalogueLinkages.Where(lmdcl => lmdcl.CatalogueID == catalogue.ID).Select(lmdcl => lmdcl.LoadMetadataID); + var lmds = _activator.CoreChildProvider.AllLoadMetadatas.Where(lmd => lmdLinkage.Contains(lmd.ID)); + foreach (var lmd in lmds) { yield return new ExecuteCommandShow(_activator, lmd.ID, typeof(LoadMetadata)) { OverrideCommandName = $"Data Load ({lmd.Name})", OverrideIcon = GetImage(RDMPConcept.LoadMetadata) }; } - if (catalogue.LoadMetadatas().Length == 0) + if (!lmds.Any()) { yield return new ExecuteCommandShow(_activator, null, typeof(LoadMetadata)) { OverrideCommandName = "No Data Load", OverrideIcon = GetImage(RDMPConcept.LoadMetadata) }; diff --git a/Rdmp.Core/CommandExecution/IBasicActivateItems.cs b/Rdmp.Core/CommandExecution/IBasicActivateItems.cs index 6c84bc6988..afe089976b 100644 --- a/Rdmp.Core/CommandExecution/IBasicActivateItems.cs +++ b/Rdmp.Core/CommandExecution/IBasicActivateItems.cs @@ -173,6 +173,7 @@ public interface IBasicActivateItems CohortCreationRequest GetCohortCreationRequest(ExternalCohortTable externalCohortTable, IProject project, string cohortInitialDescription); + IProject CohortCommitProjectSelect(IProject currentProject, Project[] projects); CohortHoldoutLookupRequest GetCohortHoldoutLookupRequest(ExternalCohortTable externalCohortTable, IProject project, CohortIdentificationConfiguration cic); diff --git a/Rdmp.Core/Curation/Data/Aggregation/AggregateConfiguration.cs b/Rdmp.Core/Curation/Data/Aggregation/AggregateConfiguration.cs index 2699e206d4..4ff492e4a7 100644 --- a/Rdmp.Core/Curation/Data/Aggregation/AggregateConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Aggregation/AggregateConfiguration.cs @@ -448,7 +448,7 @@ public override string ToString() => //strip the cic section from the front Regex.Replace(Name, $@"{CohortIdentificationConfiguration.CICPrefix}\d+_?", ""); - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context,out string reason) { var cic = GetCohortIdentificationConfigurationIfAny(); if (cic == null) @@ -457,7 +457,7 @@ public bool ShouldBeReadOnly(out string reason) return false; } - return cic.ShouldBeReadOnly(out reason); + return cic.ShouldBeReadOnly(context,out reason); } /// diff --git a/Rdmp.Core/Curation/Data/Aggregation/AggregateFilterContainer.cs b/Rdmp.Core/Curation/Data/Aggregation/AggregateFilterContainer.cs index d60856b272..cf00bf96bc 100644 --- a/Rdmp.Core/Curation/Data/Aggregation/AggregateFilterContainer.cs +++ b/Rdmp.Core/Curation/Data/Aggregation/AggregateFilterContainer.cs @@ -74,9 +74,10 @@ public override Catalogue GetCatalogueIfAny() /// /// Returns true if the filter container belongs to a parent that is frozen /// + /// /// /// - public override bool ShouldBeReadOnly(out string reason) + public override bool ShouldBeReadOnly(string context, out string reason) { var cic = GetAggregate()?.GetCohortIdentificationConfigurationIfAny(); if (cic == null) @@ -85,7 +86,7 @@ public override bool ShouldBeReadOnly(out string reason) return false; } - return cic.ShouldBeReadOnly(out reason); + return cic.ShouldBeReadOnly(context, out reason); } /// diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 804c87f6f2..745fb955a4 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -14,8 +14,10 @@ using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; using Rdmp.Core.CohortCreation.Execution; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Curation.Data.Defaults; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Serialization; @@ -30,6 +32,7 @@ using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.DataAccess; +using Rdmp.Core.Startup; using Rdmp.Core.Ticketing; namespace Rdmp.Core.Curation.Data; @@ -85,7 +88,6 @@ public sealed class Catalogue : DatabaseEntity, IComparable, ICatalogue, IInject private int? _pivotCategoryExtractionInformationID; private bool _isDeprecated; private bool _isInternalDataset; - private bool _isColdStorageDataset; private int? _liveLoggingServerID; private string _shortDescription; @@ -489,15 +491,6 @@ public bool IsInternalDataset set => SetField(ref _isInternalDataset, value); } - /// - [DoNotExtractProperty] - [DoNotImportDescriptions] - public bool IsColdStorageDataset - { - get => _isColdStorageDataset; - set => SetField(ref _isColdStorageDataset, value); - } - /// [Relationship(typeof(ExternalDatabaseServer), RelationshipType.LocalReference)] [DoNotExtractProperty] @@ -559,6 +552,7 @@ public DateTime? DatasetStartDate /// public LoadMetadata[] LoadMetadatas() { + var loadMetadataLinkIDs = Repository.GetAllObjectsWhere("CatalogueID", ID).Select(l => l.LoadMetadataID); return Repository.GetAllObjects().Where(cat => loadMetadataLinkIDs.Contains(cat.ID)).ToArray(); @@ -1103,7 +1097,6 @@ internal Catalogue(ICatalogueRepository repository, DbDataReader r) Source_URL = ParseUrl(r, "Source_URL"); IsDeprecated = (bool)r["IsDeprecated"]; IsInternalDataset = (bool)r["IsInternalDataset"]; - IsColdStorageDataset = (bool)r["IsColdStorageDataset"]; Folder = r["Folder"].ToString(); @@ -1147,6 +1140,25 @@ internal Catalogue(ShareManager shareManager, ShareDefinition shareDefinition) /// public override string ToString() => Name; + public List GetLinkedDatasets() + { + return CatalogueRepository.GetAllObjectsWhere("Catalogue_ID", this.ID).Select(l => l.Dataset).Distinct().ToList(); + } + + public override void SaveToDatabase() + { + base.SaveToDatabase(); + foreach (var dataset in CatalogueRepository.GetAllObjectsWhere("Catalogue_ID", this.ID).Where(cdl => cdl.Autoupdate).Select(cld => cld.Dataset)) + { + var provider = CatalogueRepository.GetObjectByID((int)dataset.Provider_ID); + var providerConfiguration = CatalogueRepository.GetObjectByID((int)dataset.Provider_ID); + var repositoryProvider = new UserSettingsRepositoryFinder(); + var activator = new ThrowImmediatelyActivator(repositoryProvider, ThrowImmediatelyCheckNotifier.Quiet); + var providerInstance = providerConfiguration.GetProviderInstance(activator); + providerInstance.UpdateUsingCatalogue(dataset, this); + } + } + /// /// Sorts alphabetically based on /// diff --git a/Rdmp.Core/Curation/Data/Cohort/CohortAggregateContainer.cs b/Rdmp.Core/Curation/Data/Cohort/CohortAggregateContainer.cs index a0d7314bec..65ae9c2a77 100644 --- a/Rdmp.Core/Curation/Data/Cohort/CohortAggregateContainer.cs +++ b/Rdmp.Core/Curation/Data/Cohort/CohortAggregateContainer.cs @@ -203,7 +203,7 @@ public override void DeleteInDatabase() /// public override string ToString() => Name; - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context, out string reason) { var cic = GetCohortIdentificationConfiguration(); @@ -213,7 +213,7 @@ public bool ShouldBeReadOnly(out string reason) return false; } - return cic.ShouldBeReadOnly(out reason); + return cic.ShouldBeReadOnly(context,out reason); } /// diff --git a/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs b/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs index e6f9bf535c..86596eb5a5 100644 --- a/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs +++ b/Rdmp.Core/Curation/Data/Cohort/CohortIdentificationConfiguration.cs @@ -64,6 +64,7 @@ public int? Version private DateTime? _frozenDate; private int? _clonedFrom_ID; private string _folder; + private bool _isTemplate; /// [Unique] @@ -171,6 +172,15 @@ public string Folder set => SetField(ref _folder, FolderHelper.Adjust(value)); } + /// + /// Marks if CIC is used like a template, and cannot be committed + /// + public bool IsTemplate + { + get => _isTemplate; + set => SetField(ref _isTemplate, value); + } + #endregion #region Relationships @@ -234,6 +244,7 @@ internal CohortIdentificationConfiguration(ICatalogueRepository repository, DbDa ClonedFrom_ID = ObjectToNullableInt(r["ClonedFrom_ID"]); Version = ObjectToNullableInt(r["Version"]); Folder = r["Folder"] as string ?? FolderHelper.Root; + IsTemplate = Convert.ToBoolean(r["IsTemplate"]); } /// @@ -272,8 +283,14 @@ public void CreateRootContainerIfNotExists() /// public override string ToString() => Name; - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context, out string reason) { + if (IsTemplate && context == "ExecuteCommandRename") + { + reason = null; + return false; + } + if (Frozen) { reason = $"{Name} is Frozen"; diff --git a/Rdmp.Core/Curation/Data/ConcreteContainer.cs b/Rdmp.Core/Curation/Data/ConcreteContainer.cs index d93919c48a..bb1d5d5f52 100644 --- a/Rdmp.Core/Curation/Data/ConcreteContainer.cs +++ b/Rdmp.Core/Curation/Data/ConcreteContainer.cs @@ -154,7 +154,7 @@ private List GetAllSubContainersRecursively(IContainer current) } /// - public abstract bool ShouldBeReadOnly(out string reason); + public abstract bool ShouldBeReadOnly(string context, out string reason); public abstract IContainer DeepCloneEntireTreeRecursivelyIncludingFilters(); diff --git a/Rdmp.Core/Curation/Data/ConcreteFilter.cs b/Rdmp.Core/Curation/Data/ConcreteFilter.cs index 2a7000aac6..20cd045bf5 100644 --- a/Rdmp.Core/Curation/Data/ConcreteFilter.cs +++ b/Rdmp.Core/Curation/Data/ConcreteFilter.cs @@ -169,9 +169,9 @@ public virtual void Check(ICheckNotifier notifier) } /// - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context,out string reason) { reason = null; - return FilterContainer?.ShouldBeReadOnly(out reason) ?? false; + return FilterContainer?.ShouldBeReadOnly(context,out reason) ?? false; } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs b/Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs new file mode 100644 index 0000000000..6db9794192 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/CatalogueDatasetLinkage.cs @@ -0,0 +1,62 @@ +// Copyright (c) The University of Dundee 2025-2025 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.Repositories; +using System.Collections.Generic; +using System.Data.Common; + +namespace Rdmp.Core.Curation.Data.Datasets +{ + /// + /// Linkage between a and a . This is used to link datasets to catalogues in the RDMP system. + /// + public class CatalogueDatasetLinkage: DatabaseEntity + { + private int _catalogueID; + private int _datasetID; + private ICatalogueRepository _repository; + + #region Relationships + [NoMappingToDatabase] + public Catalogue Catalogue => _repository.GetObjectByID(_catalogueID); + + + [NoMappingToDatabase] + public Dataset Dataset => _repository.GetObjectByID(_datasetID); + + #endregion + + public int Catalogue_ID { get => _catalogueID; set => SetField(ref _catalogueID, value); } + + public int Dataset_ID { get => _datasetID; set => SetField(ref _datasetID, value); } + + public bool Autoupdate { get; set; } + + public CatalogueDatasetLinkage() { } + + public CatalogueDatasetLinkage(ICatalogueRepository repository, Catalogue catalogue, Dataset dataset, bool autoupdate = false) + { + _repository = repository; + _catalogueID = catalogue.ID; + _datasetID = dataset.ID; + repository.InsertAndHydrate(this, new Dictionary + { + {"Catalogue_ID", catalogue.ID }, + { "Dataset_ID", dataset.ID}, + {"Autoupdate", autoupdate==true?1:0 } + }); + } + + public CatalogueDatasetLinkage(ICatalogueRepository repository, DbDataReader r) : base(repository, r) + { + _repository = repository; + _catalogueID = int.Parse(r["Catalogue_ID"].ToString()); + _datasetID = int.Parse(r["Dataset_ID"].ToString()); + Autoupdate = r["Autoupdate"].ToString() != "0"; + } + } +} diff --git a/Rdmp.Core/Curation/Data/Dataset.cs b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs similarity index 68% rename from Rdmp.Core/Curation/Data/Dataset.cs rename to Rdmp.Core/Curation/Data/Datasets/Dataset.cs index 5623568db7..f56d590b1d 100644 --- a/Rdmp.Core/Curation/Data/Dataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/Dataset.cs @@ -11,17 +11,21 @@ using System.Data.Common; using Rdmp.Core.MapsDirectlyToDatabaseTable; using System.Diagnostics.CodeAnalysis; +using System.Linq; -namespace Rdmp.Core.Curation.Data; +namespace Rdmp.Core.Curation.Data.Datasets; /// -public sealed class Dataset : DatabaseEntity, IDataset, IHasFolder +public class Dataset : DatabaseEntity, IDataset, IHasFolder { private string _name; private string _digitalObjectIdentifier; private string _source; private string _folder = FolderHelper.Root; + private string _type = null; + private string _url = null; + private int? _providerId = null; /// [DoNotImportDescriptions] @@ -53,8 +57,37 @@ public string Source set => SetField(ref _source, value); } + public string Type + { + get => _type; + set => SetField(ref _type, value); + } + public string Url + { + get => _url; + set => SetField(ref _url, value); + } + + public int? Provider_ID + { + get => _providerId; + set => SetField(ref _providerId, value); + } public override string ToString() => Name; + public virtual string GetID() + { + return ID.ToString(); + } + public virtual string GetRemoteID() + { + return ID.ToString(); + } + + public List GetLinkedCatalogues() + { + return CatalogueRepository.GetAllObjectsWhere("Dataset_ID", this.ID).Select(l => l.Catalogue).Distinct().ToList(); + } public Dataset(ICatalogueRepository catalogueRepository, string name) { @@ -75,5 +108,11 @@ internal Dataset(ICatalogueRepository repository, DbDataReader r) DigitalObjectIdentifier = r["DigitalObjectIdentifier"].ToString(); if (r["Source"] != DBNull.Value) Source = r["Source"].ToString(); + if (r["Type"] != DBNull.Value) + Type = r["Type"].ToString(); + if (r["Url"] != DBNull.Value) + Url = r["Url"].ToString(); + if (r["Provider_ID"] != DBNull.Value) + Provider_ID = int.Parse(r["Provider_ID"].ToString()); } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs new file mode 100644 index 0000000000..1e9d637fd0 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/DatasetProviderConfiguration.cs @@ -0,0 +1,98 @@ +using MongoDB.Driver; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets; + +/// +public class DatasetProviderConfiguration : DatabaseEntity, IDatasetProviderConfiguration +{ + + private string _type; + private string _name; + private string _url; + private int _dataAccessCredentials; + private string _organisationId; + + public DatasetProviderConfiguration() { } + + public DatasetProviderConfiguration(ICatalogueRepository repository, string name, string type, string url, int dataAccessCredentialsID, string organisationId) + { + repository.InsertAndHydrate(this, new Dictionary + { + {"Name",name }, + {"Type",type}, + {"Url",url}, + {"DataAccessCredentials_ID",dataAccessCredentialsID }, + {"Organisation_ID", organisationId } + }); + } + + internal DatasetProviderConfiguration(ICatalogueRepository repository, DbDataReader r) : base(repository, r) + { + Name = r["Name"].ToString(); + Type = r["Type"].ToString(); + Url = r["Url"].ToString(); + DataAccessCredentials_ID = int.Parse(r["DataAccessCredentials_ID"].ToString()); + Organisation_ID = r["Organisation_ID"].ToString(); + } + + + public IDatasetProvider GetProviderInstance(IBasicActivateItems activator) + { + var assemblies = AppDomain.CurrentDomain.GetAssemblies(); + foreach (var assembly in assemblies) + { + var type = assembly.GetTypes().FirstOrDefault(t => t.FullName == Type); + if (type != null) + { + return (IDatasetProvider)Activator.CreateInstance(type, activator, this,null); + } + } + return null; + } + + public override string ToString() + { + return _name; + } + + public string Name + { + get => _name; + set => SetField(ref _name, value); + } + + + public string Type + { + get => _type; + set => SetField(ref _type, value); + } + + + public string Url + { + get => _url; + set => SetField(ref _url, value); + } + + public int DataAccessCredentials_ID + { + get => _dataAccessCredentials; + set => SetField(ref _dataAccessCredentials, value); + } + + public string Organisation_ID + { + get => _organisationId; + set => SetField(ref _organisationId, value); + } +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/IDataset.cs b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs similarity index 72% rename from Rdmp.Core/Curation/Data/IDataset.cs rename to Rdmp.Core/Curation/Data/Datasets/IDataset.cs index a7853f2994..3b73096298 100644 --- a/Rdmp.Core/Curation/Data/IDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs @@ -7,7 +7,7 @@ using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Repositories; -namespace Rdmp.Core.Curation.Data; +namespace Rdmp.Core.Curation.Data.Datasets; /// /// The core of datasets within RDMP. @@ -20,6 +20,26 @@ public interface IDataset: IMapsDirectlyToDatabaseTable /// ICatalogueRepository CatalogueRepository { get; } + /// + /// The Name of the Dataset + /// string Name { get; } + + /// + /// The (optional) DOI of the dataset + /// string DigitalObjectIdentifier { get; } + + /// + /// The Source of the Dataset e.g. Pure, HDR + /// + string Type { get; } + + /// + /// The URL to access the dataset + /// + string Url { get; } + + public abstract string GetID(); + public abstract string GetRemoteID(); } diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs new file mode 100644 index 0000000000..017d216784 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProvider.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; + +namespace Rdmp.Core.Curation.Data.Datasets; +/// +/// Base interface for dataset providers to impliment +/// +public interface IDatasetProvider +{ + + /// + /// Fetch a specific dataset by its ID + /// + /// + /// + Dataset FetchDatasetByID(int id); + void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue); + Dataset Create(Catalogue catalogue); + + Dataset AddExistingDatasetWithReturn(string name, string url); + void AddExistingDataset(string name, string url); + + string GetRemoteURL(Dataset dataset); +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs b/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs new file mode 100644 index 0000000000..0b7fb5a9b2 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/IDatasetProviderConfiguration.cs @@ -0,0 +1,37 @@ +using Rdmp.Core.MapsDirectlyToDatabaseTable; + +namespace Rdmp.Core.Curation.Data.Datasets; + + +/// +/// Remords configuration for accessing external dataset providers, such as PURE +/// +public interface IDatasetProviderConfiguration : IMapsDirectlyToDatabaseTable +{ + /// + /// The assembly name of the provider this configuration is used for + /// + public string Type { get; } + + /// + /// A friendly name for the configuration + /// + public string Name { get; } + + /// + /// The API url + /// + public string Url { get; } + + /// + /// Reference to which credentials should be used to access the API + /// + public int DataAccessCredentials_ID { get; } + + /// + /// The organisation ID to use with the remote provider + /// + public string Organisation_ID { get; } + + +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs new file mode 100644 index 0000000000..420c7e8e0d --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/InternalDatasetProvider.cs @@ -0,0 +1,49 @@ +using Rdmp.Core.CommandExecution; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; + +namespace Rdmp.Core.Curation.Data.Datasets; + +/// +/// Provider for internal datasets +/// +public class InternalDatasetProvider : IDatasetProvider +{ + private readonly IBasicActivateItems _activator; + public InternalDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration = null, HttpClient client = null) + { + _activator = activator; + } + + public void AddExistingDataset(string name, string url) + { + throw new System.NotImplementedException(); + } + + public Dataset AddExistingDatasetWithReturn(string name, string url) + { + throw new System.NotImplementedException(); + } + + public Dataset Create(Catalogue catalogue) + { + throw new System.NotImplementedException(); + } + + /// + public Curation.Data.Datasets.Dataset FetchDatasetByID(int id) + { + return _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ID", id).FirstOrDefault(); + } + + public string GetRemoteURL(Dataset dataset) + { + throw new System.NotImplementedException(); + } + + public void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) + { + throw new System.NotImplementedException(); + } +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs new file mode 100644 index 0000000000..887b68e403 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs @@ -0,0 +1,27 @@ +using NPOI.OpenXmlFormats.Dml; +using Rdmp.Core.Repositories; +using System.CodeDom; +using System.Data.Common; + + +namespace Rdmp.Core.Curation.Data.Datasets +{ + /// + /// Base class to allow all plugin dataset types to be based off + /// + public class PluginDataset : Rdmp.Core.Curation.Data.Datasets.Dataset + { + public PluginDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name) { } + public PluginDataset() { } + + public PluginDataset(ICatalogueRepository repository, DbDataReader r) : base(repository, r) { } + + public override string GetID() { + return ID.ToString(); + } + public override string GetRemoteID() + { + return ID.ToString(); + } + } +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs new file mode 100644 index 0000000000..32eb9970bf --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -0,0 +1,48 @@ +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets; + +public abstract class PluginDatasetProvider : IDatasetProvider +{ + protected DatasetProviderConfiguration Configuration { get; } + protected ICatalogueRepository Repository { get; } + protected IBasicActivateItems Activator { get; } + + protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration, HttpClient client = null) + { + Configuration = configuration; + Activator = activator; + Repository = activator.RepositoryLocator.CatalogueRepository; + } + + public abstract Dataset FetchDatasetByID(int id); + + public abstract Dataset AddExistingDatasetWithReturn(string name, string url); + + + public abstract void Update(string uuid, PluginDataset datasetUpdates); + + public abstract void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue); + + public virtual Dataset Create(Catalogue catalogue) + { + throw new NotImplementedException(); + } + + public virtual void AddExistingDataset(string name, string url) + { + throw new NotImplementedException(); + } + + public virtual string GetRemoteURL(Dataset dataset) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/ICatalogue.cs b/Rdmp.Core/Curation/Data/ICatalogue.cs index af2b8c5b24..4e6632b3fe 100644 --- a/Rdmp.Core/Curation/Data/ICatalogue.cs +++ b/Rdmp.Core/Curation/Data/ICatalogue.cs @@ -77,13 +77,6 @@ public interface ICatalogue : IHasDependencies, IHasQuerySyntaxHelper, INamed, I /// bool IsInternalDataset { get; set; } - /// - /// Bit flag indicating whether the Catalogue is a seldom used dataset that should be hidden by default. Use this if you are importing lots of researcher - /// datasets for cohort generation / extraction but don't want them to clog up your user interface. - /// - bool IsColdStorageDataset { get; set; } - - /// /// User specified free text field. Not used for anything by RDMP. /// diff --git a/Rdmp.Core/Curation/Data/IMightBeReadOnly.cs b/Rdmp.Core/Curation/Data/IMightBeReadOnly.cs index efc344e9dd..13716b7948 100644 --- a/Rdmp.Core/Curation/Data/IMightBeReadOnly.cs +++ b/Rdmp.Core/Curation/Data/IMightBeReadOnly.cs @@ -17,5 +17,5 @@ public interface IMightBeReadOnly /// Returns true if changes to the container should be forbidden e.g. because the parent object is frozen (like ) /// /// - bool ShouldBeReadOnly(out string reason); + bool ShouldBeReadOnly(string context,out string reason); } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilterContainer.cs b/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilterContainer.cs index 78baa0b4ea..61b07b763e 100644 --- a/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilterContainer.cs +++ b/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilterContainer.cs @@ -54,7 +54,7 @@ public SpontaneouslyInventedFilterContainer(MemoryCatalogueRepository repo, ICon public override Catalogue GetCatalogueIfAny() => null; - public override bool ShouldBeReadOnly(out string reason) + public override bool ShouldBeReadOnly(string context, out string reason) { reason = null; return false; diff --git a/Rdmp.Core/Curation/FilterImporting/ParameterCollectionUIOptions.cs b/Rdmp.Core/Curation/FilterImporting/ParameterCollectionUIOptions.cs index 2bb0a77da0..b351ab9295 100644 --- a/Rdmp.Core/Curation/FilterImporting/ParameterCollectionUIOptions.cs +++ b/Rdmp.Core/Curation/FilterImporting/ParameterCollectionUIOptions.cs @@ -50,7 +50,7 @@ public ParameterCollectionUIOptions(string useCase, ICollectSqlParameters collec _createNewParameterDelegate = CreateNewParameterDefaultImplementation; if (collector is IMightBeReadOnly ro) - ReadOnly = ro.ShouldBeReadOnly(out _); + ReadOnly = ro.ShouldBeReadOnly(this.GetType().Name, out _); } diff --git a/Rdmp.Core/Curation/KeywordHelp.txt b/Rdmp.Core/Curation/KeywordHelp.txt index aa5a3742b6..63cccdfb8b 100644 --- a/Rdmp.Core/Curation/KeywordHelp.txt +++ b/Rdmp.Core/Curation/KeywordHelp.txt @@ -99,4 +99,7 @@ FK_Redaction_RedactionConfiguration_ID: Prevents you from deleting a redaction c FK_Redaction_ColumnInfo_ID: Prevents redactions from becoming orphaned from their associated catalogue columns FK_RedactionKey_Redaction_ID: Prevents redaction primary keys from being orphaned from the associated redaction FK_RedactionKey_ColumnInfo_ID: Prevents redaction keys from becoming orphaned from their associated catalogue columns -FK_LoadMetadataRootReference: Links versions of a load metadata back to the source load metadata \ No newline at end of file +FK_LoadMetadataRootReference: Links versions of a load metadata back to the source load metadata +FK_CatalogueDatasetLinkage_Catalogue_ID:Linkage to Catalogue +FK_CatalogueDatasetLinkage_Dataset_ID: Linkage ot Dataset +FK_DatasetProviderConfiguration_DataAccessCredentials: Linkage to DataAccess Credentials diff --git a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs index cf7cdf7748..439fc505d2 100644 --- a/Rdmp.Core/Curation/SimpleStringValueEncryption.cs +++ b/Rdmp.Core/Curation/SimpleStringValueEncryption.cs @@ -22,21 +22,23 @@ public class SimpleStringValueEncryption : IEncryptStrings private readonly RSACryptoServiceProvider _turing = new(); private const string Key = - @" - - AQAB - sMDeszVErUmbqOxQavw5OsWpL3frccEGtTJYM8G54Fw7NK6xFVUrq79nWB6px4/B -

6kcXnTVJrVuD9j6qUm+F71jIL2H92lgN

- wSRbrdj1qGBPBnYMO5dx11gvfNCKKdWF - aKdxaQzQ6Nwkyu+bbk/baNwkMOZ5W/xR - B/B8rErM3l0HIpbbrd9t2JJRcWoJI+sZ - NFv4Z26nbMpOkOcAnO3rktoMffza+3Ul - Y8zC8dUF7gI9zeeAkKfReInauV6wpg4iVh7jaTDN5DAmKFURTAyv6Il6LEyr07JB -"; + @" + + xZ7V6twqlj+3L1hsjB+BwhzQEgcs8tpqzgSjwgjPwzfisLFBN9HkwX2u+ZrgrKhKmZgSOVBYRhZiGJeF1hY5gLz3/Zo05fcvtgL0ylEhi6wnH9y6CPu8+4xCOJ26eywtNS02v2CC/0HUWnw6kNHBqdyyNjwuC12Ll5YJ3wZ1kQZ2mce3DMjpgRDvYqnm8ldwLWpgreK7I5vggQgons3v39x+Cx501Eo0qd+iHzXf6/8pUhYqb+xhDdb9gnHcitipXVFG3Ts27OUiuO0uLAO4Es400ApyWpvHdCCNrBJ/EQWZGEO/qVKmfP0CWBmokVL+IUawX9lPl/Luo9W5AFpaIQ== + AQAB +

xvQflEOfLYAXiMFrG/alxcgUbimbY8Qc1/3Jm21CFVrkQL1rNQ/PEXavXYyNP3jRwUrqKbJrtcdz5MPgKpvA/iWdqjp4Qx+V7N2DkdzKL7liViKoCHf1eXfyeWkCErHOgInTerOSlCaeABqkqpr+eaAtzy2j4df5U/1+j8g7I48=

+ /kjapfiD7aC1yQX8BYiJ29oTlvZx0sptfXA0Bx9qXvd1UFTqjDbVoIF7tXk+DsottRL97G52ImpYf4w4wJFb1F+d5sQCaCrEzIdQ/4mJdpn7FLYxErYXpoZl0e2knIoqQie7+vaw2oV9YSx3iJHoEVfDM+0NAW+aCUUJGNorD08= + mAYms0ZQtZXxZcBWNhHsbgsLAXqtkDhkye7VRPzhyCuhyo5zAyLHWVLVgahKrjuGHCtAbwg1Ibv8pMu/2Q8XE5xus4rmJnRWPZ6uUKDjpkAEEkl9GKuBWYX8NCW3Pc28O6AVhub8lFRF21KAjRTOauWo22zGk2ZS0IkdUoTwG6U= + nQsrllNMT0bw3k0G3/f6hEBD1vkvROrmAhF44GlDjZEw78Lx9FStTOqLF4HglMvCvNEU55808H5TV7qnFi7v0tKWt32YqvK3BkYP/THZJtlkWt9GoXK6WosoeSVWg6NFBAR8MTuH7/1/eLM4w6yw8X0NPpWJcbiWHmF3g9TBwTs= + fbI4APSSbOOd07goBWzZYjdrEVEZDiboK4jFKYoMI19PrbQBTvpRmLsJxhg96fgjiFfmw05tsPIsLSQPczsav6JDq25sQyOaz7VE33Y8CEV4rVvI+l7vlWqvn/EfThPqYdR4UOHa6G5wcBTI48O4+SmKMhLQl1GK/ZA+XEmlHx8= + QC+Uv1F/K4nKT8BikShylr+Q/SoDeWVjp0Juhcki4f82y7jmu+CachYGTN/29V07zaNM1/y2jx0aA27Dc4OIbb3ythXt9HtSrcVMCKJNSPZDRuAENIK/INyvbYAdX4A7trfWvlX0dj/FXxZWV08pnagm4eKt+dcKTdPXpO6OJOnnSYMcdRBFdZLAj/sh8oMNXMHSxsCA7YHjL+4NGbpyTMYnfa6CUxFc36cLi2/Hm6JhV1nYEswTI5T1qRqmWpF1+H0ksxLKx6I5D61mzfaf7RQPJRCLhddXoLPIh8IaoNIomPcZyZQBxwBhtgj8pf9rcBN50aqNc5V2vKoEOt/gGQ== + "; + + private readonly string _parameters; public SimpleStringValueEncryption(string parameters) { - _turing.FromXmlString(parameters ?? Key); + _parameters=parameters; } /// @@ -46,6 +48,7 @@ public SimpleStringValueEncryption(string parameters) /// public string Encrypt(string toEncrypt) { + _turing.FromXmlString(_parameters ?? Key); // Fall back on bad encryption if no private key is configured if (_turing.KeySize < 1024) return string.Join('-', @@ -70,6 +73,7 @@ public string Encrypt(string toEncrypt) /// public string Decrypt(string toDecrypt) { + _turing.FromXmlString(_parameters ?? Key); if (toDecrypt.StartsWith("$js1$", StringComparison.Ordinal) && toDecrypt.EndsWith("$", StringComparison.Ordinal)) { // Good, it's a new-style AES+RSA encrypted string diff --git a/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs b/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs index 0c164559ff..eae90ae836 100644 --- a/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs +++ b/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs @@ -220,12 +220,7 @@ public IReleaseLog[] ReleaseLog /// [NoMappingToDatabase] - public IPipeline DefaultPipeline => - DefaultPipeline_ID == null - ? null - : (IPipeline)((IDataExportRepository)Repository).CatalogueRepository.GetObjectByID( - DefaultPipeline_ID.Value); - + public IPipeline DefaultPipeline => DefaultPipeline_ID == null?null: ((IDataExportRepository) Repository).CatalogueRepository.GetAllObjects().FirstOrDefault(p => p.ID == DefaultPipeline_ID); /// [NoMappingToDatabase] @@ -347,7 +342,7 @@ internal ExtractionConfiguration(IDataExportRepository repository, DbDataReader /// public override string ToString() => Name; - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context,out string reason) { if (IsReleased) { diff --git a/Rdmp.Core/DataExport/Data/FilterContainer.cs b/Rdmp.Core/DataExport/Data/FilterContainer.cs index f5b805a4ce..2e425d3579 100644 --- a/Rdmp.Core/DataExport/Data/FilterContainer.cs +++ b/Rdmp.Core/DataExport/Data/FilterContainer.cs @@ -51,7 +51,7 @@ public override Catalogue GetCatalogueIfAny() return (Catalogue)sel?.ExtractableDataSet.Catalogue; } - public override bool ShouldBeReadOnly(out string reason) + public override bool ShouldBeReadOnly(string context, out string reason) { var ec = GetSelectedDataSetsRecursively()?.ExtractionConfiguration; @@ -61,7 +61,7 @@ public override bool ShouldBeReadOnly(out string reason) return false; } - return ec.ShouldBeReadOnly(out reason); + return ec.ShouldBeReadOnly(context, out reason); } /// diff --git a/Rdmp.Core/DataExport/Data/Project.cs b/Rdmp.Core/DataExport/Data/Project.cs index 9855e434c3..f03690e790 100644 --- a/Rdmp.Core/DataExport/Data/Project.cs +++ b/Rdmp.Core/DataExport/Data/Project.cs @@ -176,7 +176,14 @@ public CohortIdentificationConfiguration[] GetAssociatedCohortIdentificationConf { var associations = Repository.GetAllObjectsWithParent(this); - return associations.Select(a => a.CohortIdentificationConfiguration).Where(c => c != null).ToArray(); + return associations.Select(a => a.CohortIdentificationConfiguration).Where(c => c != null && !c.IsTemplate).ToArray(); + } + + public CohortIdentificationConfiguration[] GetAssociatedTemplateCohortIdentificationConfigurations() + { + var associations = + Repository.GetAllObjectsWithParent(this); + return associations.Select(a => a.CohortIdentificationConfiguration).Where(c => c != null && c.IsTemplate).ToArray(); } /// diff --git a/Rdmp.Core/DataExport/Data/ProjectCohortIdentificationConfigurationAssociation.cs b/Rdmp.Core/DataExport/Data/ProjectCohortIdentificationConfigurationAssociation.cs index 4a462ff0ed..fe4ad090d5 100644 --- a/Rdmp.Core/DataExport/Data/ProjectCohortIdentificationConfigurationAssociation.cs +++ b/Rdmp.Core/DataExport/Data/ProjectCohortIdentificationConfigurationAssociation.cs @@ -122,10 +122,10 @@ public override string ToString() return assoc == null ? "Orphan Association" : assoc.Name; } - public bool ShouldBeReadOnly(out string reason) + public bool ShouldBeReadOnly(string context, out string reason) { reason = null; - return CohortIdentificationConfiguration?.ShouldBeReadOnly(out reason) ?? false; + return CohortIdentificationConfiguration?.ShouldBeReadOnly(context,out reason) ?? false; } /// diff --git a/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs b/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs index 19884744de..42df873bf8 100644 --- a/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs +++ b/Rdmp.Core/DataExport/Data/ProjectSpecificCatalogueManager.cs @@ -62,11 +62,11 @@ public static ExtractableDataSet MakeCatalogueProjectSpecific(IDataExportReposit return eds; } - public static bool CanMakeCatalogueNonProjectSpecific(IDataExportRepository dqeRepo, ICatalogue catalogue, ExtractableDataSet extractableDataSet,IProject project) + public static bool CanMakeCatalogueNonProjectSpecific(IDataExportRepository dqeRepo, ICatalogue catalogue, ExtractableDataSet extractableDataSet, IProject project) { bool usedInExtraction = project.ExtractionConfigurations.Where(ec => ec.SelectedDataSets.Select(sds => sds.ExtractableDataSet).Contains(extractableDataSet)).Any(); if (!usedInExtraction) return true; - if(usedInExtraction && extractableDataSet.Projects.Count ==1) return true; + if (usedInExtraction && extractableDataSet.Projects.Count == 1) return true; return false; } @@ -82,7 +82,7 @@ public static void MakeCatalogueNonProjectSpecific(IDataExportRepository dqeRepo p.DeleteInDatabase(); } extractableDataSet.Projects.Remove(project); - + foreach (var ei in catalogue.GetAllExtractionInformation(ExtractionCategory.ProjectSpecific)) { ei.ExtractionCategory = ExtractionCategory.Core; @@ -91,4 +91,3 @@ public static void MakeCatalogueNonProjectSpecific(IDataExportRepository dqeRepo } } } - diff --git a/Rdmp.Core/DataExport/Data/SelectedDataSets.cs b/Rdmp.Core/DataExport/Data/SelectedDataSets.cs index 5c8ed8ab66..402b137168 100644 --- a/Rdmp.Core/DataExport/Data/SelectedDataSets.cs +++ b/Rdmp.Core/DataExport/Data/SelectedDataSets.cs @@ -126,7 +126,7 @@ public SelectedDataSets(IDataExportRepository repository, ExtractionConfiguratio /// public override string ToString() => ExtractableDataSet.ToString(); - public bool ShouldBeReadOnly(out string reason) => ExtractionConfiguration.ShouldBeReadOnly(out reason); + public bool ShouldBeReadOnly(string context, out string reason) => ExtractionConfiguration.ShouldBeReadOnly(context, out reason); /// public string GetDeleteVerb() => "Remove"; diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs index 714b6edf44..3b67f5332f 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs @@ -4,12 +4,6 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text.RegularExpressions; using FAnsi.Discovery; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; @@ -18,7 +12,10 @@ using Rdmp.Core.DataExport.DataRelease.Pipeline; using Rdmp.Core.DataExport.DataRelease.Potential; using Rdmp.Core.DataFlowPipeline; +using Rdmp.Core.DataLoad.Engine.Job.Scheduling; using Rdmp.Core.DataLoad.Engine.Pipeline.Destinations; +using Rdmp.Core.DataLoad.Triggers.Exceptions; +using Rdmp.Core.DataLoad.Triggers.Implementations; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; @@ -26,6 +23,12 @@ using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.DataAccess; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; using YamlDotNet.Core; namespace Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; @@ -109,6 +112,10 @@ You must have either $a or $d [DemandsInitialization("An optional list of columns to index on e.g \"Column1, Column2\"")] public string UserDefinedIndex { get; set; } + + [DemandsInitialization("When writing to an existing database, will update records and store historical versions via a view", DefaultValue = false)] + public bool UseArchiveTrigger { get; set; } + private DiscoveredDatabase _destinationDatabase; private DataTableUploadDestination _destination; @@ -177,6 +184,7 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis var existing = _destinationDatabase.ExpectTable(tblName); if (existing.Exists()) { + var hasPKs = existing.DiscoverColumns().Any(col => col.IsPrimaryKey); if (_request.IsBatchResume) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, @@ -194,6 +202,26 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis // since we dropped it we should treat it as if it was never there to begin with _tableDidNotExistAtStartOfLoad = true; } + else if (UseArchiveTrigger && hasPKs) + { + + TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer); + var implementor = triggerFactory.Create(existing); + bool present; + try + { + present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + } + catch (TriggerMissingException) + { + present = false; + } + if (!present) + { + implementor.CreateTrigger(ThrowImmediatelyCheckNotifier.Quiet); + } + + } else { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, @@ -223,11 +251,13 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis _destination.IncludeTimeStamp = IncludeTimeStamp; _destination.UseTrigger = AppendDataIfTableExists; _destination.IndexTables = IndexTables; + _destination.UseTrigger = UseArchiveTrigger; _destination.IndexTableName = GetIndexName(); if (UserDefinedIndex is not null) _destination.UserDefinedIndexes = UserDefinedIndex.Split(',').Select(i => i.Trim()).ToList(); _destination.PreInitialize(_destinationDatabase, listener); + return _destination; } @@ -343,7 +373,7 @@ private string GetIndexName() } - return indexName.Replace(" ",""); + return indexName.Replace(" ", ""); } private string GetTableName(string suffix = null) @@ -664,6 +694,8 @@ public override void Check(ICheckNotifier notifier) $"Catalogue '{dsRequest.Catalogue}' does not have an Acronym but TableNamingPattern contains $a", CheckResult.Fail)); + + base.Check(notifier); try @@ -671,6 +703,24 @@ public override void Check(ICheckNotifier notifier) var server = DataAccessPortal.ExpectServer(TargetDatabaseServer, DataAccessContext.DataExport, false); var database = _destinationDatabase = server.ExpectDatabase(GetDatabaseName()); + if (UseArchiveTrigger) + { + if (_request is ExtractDatasetCommand dsRequest) + { + var existing = _destinationDatabase.ExpectTable(dsRequest.Catalogue.Name); + if (existing.Exists()) + { + var hasPKs = existing.DiscoverColumns().Any(col => col.IsPrimaryKey); + if (!hasPKs) + { + notifier.OnCheckPerformed(new CheckEventArgs( + $"Catalogue does not have any PKS. Cannot apply the archive trigger", + CheckResult.Fail)); + } + } + } + } + if (database.Exists()) { notifier.OnCheckPerformed( diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/ExtractionPipelineUseCase.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/ExtractionPipelineUseCase.cs index 87f95b82a9..faa03834e6 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/ExtractionPipelineUseCase.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/ExtractionPipelineUseCase.cs @@ -316,6 +316,19 @@ private void WriteMetadata(IDataLoadEventListener listener) new NotifyEventArgs(ProgressEventType.Error, "Word metadata document NOT CREATED", e)); return; } + try + { + var datasetVariableReportWriter = new DatasetVariableReportGenerator(this); + datasetVariableReportWriter.GenerateDatasetVariableReport(); + } + catch (Exception e) + { + ExtractCommand.ElevateState(ExtractCommandState.Warning); + + listener.OnNotify(this, + new NotifyEventArgs(ProgressEventType.Error, "Dataset variable document NOT CREATED", e)); + return; + } //if there were any exceptions if (wordDataWriter.ExceptionsGeneratingWordFile.Any()) diff --git a/Rdmp.Core/DataExport/DataRelease/Potential/FlatFileReleasePotential.cs b/Rdmp.Core/DataExport/DataRelease/Potential/FlatFileReleasePotential.cs index fb74c32fc7..57137b6ee3 100644 --- a/Rdmp.Core/DataExport/DataRelease/Potential/FlatFileReleasePotential.cs +++ b/Rdmp.Core/DataExport/DataRelease/Potential/FlatFileReleasePotential.cs @@ -49,6 +49,7 @@ private bool FilesAreMissing(IExtractionResults extractionResults) { ExtractFile = new FileInfo(extractionResults.DestinationDescription); var metadataFile = new FileInfo(extractionResults.DestinationDescription.Replace(".csv", ".docx")); + var variablesFile = new FileInfo(extractionResults.DestinationDescription.Replace(".csv","Variables.csv")); if (!ExtractFile.Exists) return true; //extract is missing @@ -61,7 +62,7 @@ private bool FilesAreMissing(IExtractionResults extractionResults) //see if there is any other pollution in the extract directory var unexpectedFile = ExtractFile.Directory.EnumerateFiles().FirstOrDefault(f => - !(f.Name.Equals(ExtractFile.Name) || f.Name.Equals(metadataFile.Name))); + !(f.Name.Equals(ExtractFile.Name) || f.Name.Equals(metadataFile.Name) || f.Name.Equals(variablesFile.Name))); if (unexpectedFile != null) throw new Exception( diff --git a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs index 913650fee7..9ecaca4fd1 100644 --- a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs +++ b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs @@ -4,22 +4,17 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Data; -using System.Diagnostics; -using System.Globalization; -using System.Linq; -using System.Threading.Tasks; +using FAnsi; using FAnsi.Connections; using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; using FAnsi.Discovery.TableCreation; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; -using Rdmp.Core.DataLoad.Triggers.Implementations; using Rdmp.Core.DataLoad.Triggers; +using Rdmp.Core.DataLoad.Triggers.Implementations; using Rdmp.Core.Logging; using Rdmp.Core.Logging.Listeners; using Rdmp.Core.Repositories.Construction; @@ -27,8 +22,14 @@ using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.DataAccess; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; using TypeGuesser; -using FAnsi; namespace Rdmp.Core.DataLoad.Engine.Pipeline.Destinations; @@ -145,38 +146,6 @@ private static object[] FilterOutItemAtIndex(object[] itemArray, int[] indexes) if (indexes.Length == 0) return itemArray; return itemArray.Where((source, idx) => !indexes.Contains(idx)).ToArray(); } - private string GetPKValue(DataColumn pkColumn, DataRow row) - { - var pkName = pkColumn.ColumnName; - var value = row[pkName]; - if (_externalCohortTable is not null) - { - var privateIdentifierField = _externalCohortTable.PrivateIdentifierField.Split('.').Last()[1..^1];//remove the "[]" from the identifier field - var releaseIdentifierField = _externalCohortTable.ReleaseIdentifierField.Split('.').Last()[1..^1];//remove the "[]" from the identifier field - if (pkName == releaseIdentifierField) - { - //going to have to look up the previous relaseID to match - DiscoveredTable cohortTable = _externalCohortTable.DiscoverCohortTable(); - using var lookupDT = cohortTable.GetDataTable(); - var releaseIdIndex = lookupDT.Columns.IndexOf(releaseIdentifierField); - var privateIdIndex = lookupDT.Columns.IndexOf(privateIdentifierField); - var foundRow = lookupDT.Rows.Cast().Where(r => r.ItemArray[releaseIdIndex].ToString() == value.ToString()).LastOrDefault(); - if (foundRow is not null) - { - var originalValue = foundRow.ItemArray[privateIdIndex]; - var existingIDsforReleaseID = lookupDT.Rows.Cast().Where(r => r.ItemArray[privateIdIndex].ToString() == originalValue.ToString()).Select(s => s.ItemArray[releaseIdIndex].ToString()); - if (existingIDsforReleaseID.Count() > 0) - { - //we don't know what the current release ID is ( there may be ones from multiple cohorts) - var ids = existingIDsforReleaseID.Select(id => $"'{id}'"); - return $"{pkName} in ({string.Join(',', ids)})"; - } - } - } - } - return $"{pkName} = '{value}'"; - } - public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) @@ -393,29 +362,45 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener } - - foreach (DataRow row in rowsToModify.Distinct()) + if (rowsToModify.Any()) { - //replace existing - var args = new DatabaseOperationArgs(); - List columns = []; - foreach (DataColumn column in toProcess.Columns) + var tmpDt = toProcess.Clone(); + foreach (var row in rowsToModify) + { + tmpDt.ImportRow(row); + } + var tblName = $"tmpTable_{DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond}"; + var tmpTable = _discoveredTable.Database.CreateTable(tblName, tmpDt); + + var customLines = new List() { }; + foreach (var col in tmpDt.Columns.Cast()) + { + if (!pkColumns.Select(c => c.ColumnName).Contains(col.ColumnName)) + { + customLines.Add(new CustomLine($"t1.{col.ColumnName} = t2.{col.ColumnName}", QueryComponent.SET)); + } + else + { + customLines.Add(new CustomLine($"t1.{col.ColumnName} = t2.{col.ColumnName}", QueryComponent.JoinInfoJoin)); + customLines.Add(new CustomLine($"t1.{col.ColumnName} = t2.{col.ColumnName}", QueryComponent.WHERE)); + } + } + var existingColumns = _discoveredTable.DiscoverColumns().Select(c => c.GetRuntimeName()).Where(c => c != SpecialFieldNames.DataLoadRunID && c != SpecialFieldNames.ValidFrom); + var columnsThatPreviouslyExisted = existingColumns.Where(ec => !toProcess.Columns.Cast().ToList().Select(c => c.ColumnName).Contains(ec)); + foreach (var col in columnsThatPreviouslyExisted) + { + customLines.Add(new CustomLine($"t1.{col} = t2.{col}", QueryComponent.SET)); + + } + + var str = _discoveredTable.Database.Server.GetQuerySyntaxHelper().UpdateHelper.BuildUpdate(_discoveredTable, tmpTable, customLines); + using (var connection = _discoveredTable.Database.Server.GetConnection()) { - //if (!pkColumns.Contains(column)) - //{ - columns.Add(column.ColumnName); - //} + connection.Open(); + var cmd = _discoveredTable.Database.Server.GetCommand(str, connection); + cmd.ExecuteNonQuery(); + tmpTable.Drop(); } - //need to check for removed column and null them out - var existingColumns = _discoveredTable.DiscoverColumns().Select(c => c.GetRuntimeName()); - var columnsThatPreviouslyExisted = existingColumns.Where(c => !pkColumns.Select(pk => pk.ColumnName).Contains(c) && !columns.Contains(c) && c != SpecialFieldNames.DataLoadRunID && c != SpecialFieldNames.ValidFrom); - var nullEntries = string.Join(" ,", columnsThatPreviouslyExisted.Select(c => $"{c} = NULL")); - var nullText = nullEntries.Length > 0 ? $" , {nullEntries}" : ""; - var columnString = string.Join(" , ", columns.Select(col => $"{col} = '{row[col]}'").ToList()); - var pkMatch = string.Join(" AND ", pkColumns.Select(pk => GetPKValue(pk, row)).ToList()); - var sql = $"update {_discoveredTable.GetFullyQualifiedName()} set {columnString} {nullText} where {pkMatch}"; - var cmd = _discoveredTable.GetCommand(sql, args.GetManagedConnection(_discoveredTable).Connection); - cmd.ExecuteNonQuery(); } foreach (DataRow row in rowsToModify.Distinct()) diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RemoteServerSQLExecution.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RemoteServerSQLExecution.cs new file mode 100644 index 0000000000..9027c6b6a5 --- /dev/null +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RemoteServerSQLExecution.cs @@ -0,0 +1,62 @@ +using FAnsi.Discovery; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.DataLoad.Engine.Mutilators; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.DataLoad.Modules.Mutilators +{ + internal class RemoteServerSQLExecution : IPluginMutilateDataTables + { + + [DemandsInitialization("The Remote Database server to run this sql on")] + public ExternalDatabaseServer RemoteServer { get; set; } + + [DemandsInitialization("Run the following SQL when this component is run in the DLE", DemandType = DemandType.SQL, + Mandatory = true)] + public string Sql { get; set; } + + public void Check(ICheckNotifier notifier) + { + if (!RemoteServer.Exists()) + { + notifier.OnCheckPerformed(new CheckEventArgs("Remote Server unavailable", CheckResult.Fail)); + } + } + + public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage) + { + } + + public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener) + { + } + + public ExitCodeType Mutilate(IDataLoadJob job) + { + var db = RemoteServer.Discover(ReusableLibraryCode.DataAccess.DataAccessContext.DataLoad); + try + { + using (var conn = db.Server.GetConnection()) + { + conn.Open(); + var cmd = db.Server.GetCommand(Sql, conn); + cmd.ExecuteNonQuery(); + } + } + catch (Exception e) + { + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, e.Message)); + return ExitCodeType.Error; + } + return ExitCodeType.Success; + } + } +} diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql new file mode 100644 index 0000000000..3f31ee72a2 --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddDatasetUpdates.sql @@ -0,0 +1,44 @@ +--Version: 9.1.1 +--Description: Add new metadata fields for catalogues +if not exists (select 1 from sys.tables where name = 'CatalogueDatasetLinkage') +BEGIN +CREATE TABLE [dbo].[CatalogueDatasetLinkage]( + [ID] [int] IDENTITY(1,1) NOT NULL, + [Catalogue_ID] [int] NOT NULL, + [Dataset_ID] [int] NOT NULL, + [Autoupdate] [int] NOT NULL DEFAULT 0, + CONSTRAINT FK_CatalogueDatasetLinkage_Catalogue_ID FOREIGN KEY (Catalogue_ID) REFERENCES Catalogue(ID) ON DELETE CASCADE, + CONSTRAINT FK_CatalogueDatasetLinkage_Dataset_ID FOREIGN KEY (Dataset_ID) REFERENCES Dataset(ID) ON DELETE CASCADE, +CONSTRAINT [PK_CatalogueDatasetLinkage] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO + +if not exists (select 1 from sys.tables where name = 'DatasetProviderConfiguration') +BEGIN +CREATE TABLE [dbo].[DatasetProviderConfiguration]( + [ID] [int] IDENTITY(1,1) NOT NULL, + Type [nvarchar](256) NOT NULL, + Url [nvarchar](500) NOT NULL, + Name [nvarchar](256) NOT NULL, + DataAccessCredentials_ID [int] NOT NULL, + Organisation_ID [nvarchar](256) NOT NULL + CONSTRAINT FK_DatasetProviderConfiguration_DataAccessCredentials FOREIGN KEY (DataAccessCredentials_ID) REFERENCES DataAccessCredentials(ID) ON DELETE CASCADE, +CONSTRAINT [PK_DatasetProviderConfiguration] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO +if not exists (select 1 from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME='Dataset' and COLUMN_NAME='Type') +BEGIN +ALTER TABLE [dbo].[Dataset] +ADD [Type] [varchar](256) NULL, + [Url] [varchar](256) NULL, + [Provider_ID] [int] NULL +END +GO \ No newline at end of file diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql new file mode 100644 index 0000000000..4bccc079cf --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/091_AddTemplateFlagToCIC.sql @@ -0,0 +1,11 @@ +--Version: 9.1.0 +--Description: Add new metadata fields for catalogues +if not exists (select 1 from sys.columns where name = 'IsTemplate' and OBJECT_NAME(object_id) = 'CohortIdentificationConfiguration') +BEGIN +ALTER TABLE [dbo].[CohortIdentificationConfiguration] +ADD +[IsTemplate] [bit] NOT NULL DEFAULT 0 +END +GO + + diff --git a/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs b/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs new file mode 100644 index 0000000000..46858a1817 --- /dev/null +++ b/Rdmp.Core/Dataset/Confluence/ConfluencePageBuilder.cs @@ -0,0 +1,246 @@ +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Rdmp.Core.Dataset.Confluence +{ + /// + /// Used to generate HTML to create a confluence tree of + /// - Overview Page + /// - Each catalogue + /// Tabulates dataset variables and metadata about each catalogue + /// + public class ConfluencePageBuilder + { + + private readonly List _catalogues; + + private readonly string _repositoryName = ""; + private readonly string _repositoryDescription = ""; + private readonly string _subdomain = ""; + public ConfluencePageBuilder(List catalogues, string name, string description, string subdomain) + { + _catalogues = catalogues; + _repositoryName = name; + _repositoryDescription = description; + _subdomain = subdomain; + } + + private string BuildCatalogueOverviewHTML(Catalogue catalogue, string pageId) + { + Console.WriteLine($"Building overview for catalogue '{catalogue.Name}'"); + string name = pageId != null ? $"{catalogue.Name}" : catalogue.Name; + + return $""" +

{catalogue.Name}{(catalogue.Acronym != null ? $" ({catalogue.Acronym})" : "")}

+ + + + + + + + + + + + + +
+ Dataset Name + + Description + + Tags + + Acronym +
+ {name} + + {catalogue.Description} + + {string.Join(", ", catalogue.Search_keywords)} + + {catalogue.Acronym} +
+ """; + } + + public string BuildContainerPage(Dictionary cataloguePageLookup) + { + Console.WriteLine($"Building container page. Found {_catalogues.Count} catalogues."); + return $""" + +

{_repositoryName} Catalogues

+

{_repositoryDescription}

+

+ {string.Join("", _catalogues.Select(c => { cataloguePageLookup.TryGetValue(c.ID, out var pageId); return BuildCatalogueOverviewHTML(c, pageId); }))} + """; + } + + private string SplitCamelCase(string input) + { + if (input is null) return input; + return System.Text.RegularExpressions.Regex.Replace(input.Replace(",", ", "), "([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim(); + + } + + private static string BuildDataVariableRecord(CatalogueItem catalogueItem) + { + var lookups = catalogueItem.CatalogueRepository.GetAllObjectsWhere("ForeignKey_ID", catalogueItem.ColumnInfo.ID); + return $""" + + {catalogueItem.Name} + {catalogueItem.ColumnInfo.Data_type} + {catalogueItem.ExtractionInformation.IsPrimaryKey} + {catalogueItem.Description} + {catalogueItem.ExtractionInformation.IsExtractionIdentifier} + {lookups.Length != 0} + + """; + } + + public string BuildHTMLForCatalogue(Catalogue catalogue) + { + Console.WriteLine($"Building HTML for catalogue '{catalogue.Name}'"); + //todo think about splitting and spacing of some of these values + return $""" +

Dataset Summary

+

{catalogue.Description}

+

+

Data Details

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Resource Type{SplitCamelCase(Enum.GetName(catalogue.Type))}
Dataset Purpose{SplitCamelCase(Enum.GetName(catalogue.Purpose))}
Dataset Type{SplitCamelCase(catalogue.DataType)}
Dataset SubType{SplitCamelCase(catalogue.DataSubType)}
Dataset Source{SplitCamelCase(catalogue.DataSource)}
Dataset Source Setting{SplitCamelCase(catalogue.DataSourceSetting)}
Dataset Purpose{SplitCamelCase(Enum.GetName(catalogue.Purpose))}
Dataset Keywords{catalogue.Search_keywords}
+

Geospatial and Temporal Details

+ + + + + + + + + + + + + + + + + +
Geographic Coverage{catalogue.Geographical_coverage}
Granularity{catalogue.Granularity}
Start Date{catalogue.StartDate}
End Date{catalogue.EndDate}
+

Access Details

+ + + + + + + + + + + + + + + + + +
Access Contact{catalogue.Contact_details}
Data Controller{catalogue.DataController}
Data Processor{catalogue.DataProcessor}
Jurisdiction{catalogue.Juristiction}
+

Attribution

+ + + + + + + + + + + + + + + + + +
DOI{catalogue.Doi}
Controlled Vocabulary{catalogue.ControlledVocabulary}
Associated People{catalogue.AssociatedPeople}
Jurisdiction{catalogue.Juristiction}
+

Data Updates

+ + + + + + + + + + + + + +
Update Frequency{Enum.GetName(catalogue.Update_freq)}
Initial Release Date{catalogue.DatasetReleaseDate}
Update Lag{Enum.GetName(catalogue.UpdateLag)}
+

Dataset Variables

+ + + + + + + + + + {string.Join("", catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation is not null).Select(ci => BuildDataVariableRecord(ci)))} +
+ Variable Name + + Type + + Null Possible (Y/N) + + Description + + Identifier + + Has Lookups +
+ """; + } + } +} diff --git a/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs b/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs index d51360e1a2..07ee42c59f 100644 --- a/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs +++ b/Rdmp.Core/Dataset/DatasetConfigurationUICommon.cs @@ -20,7 +20,7 @@ public class DatasetConfigurationUICommon /// public IBasicActivateItems Activator; - public Curation.Data.Dataset Dataset; + public Curation.Data.Datasets.Dataset Dataset; public DatasetConfigurationUICommon() diff --git a/Rdmp.Core/Icons/AllTemplateCohortIdentificationConfigurationsNode.png b/Rdmp.Core/Icons/AllTemplateCohortIdentificationConfigurationsNode.png new file mode 100644 index 0000000000..e0cdd77c8a Binary files /dev/null and b/Rdmp.Core/Icons/AllTemplateCohortIdentificationConfigurationsNode.png differ diff --git a/Rdmp.Core/Icons/AssociatedCohortIdentificationTemplatesNode.png b/Rdmp.Core/Icons/AssociatedCohortIdentificationTemplatesNode.png new file mode 100644 index 0000000000..e0cdd77c8a Binary files /dev/null and b/Rdmp.Core/Icons/AssociatedCohortIdentificationTemplatesNode.png differ diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIconProvider.cs b/Rdmp.Core/Icons/IconProvision/CatalogueIconProvider.cs index f6db65f99a..a87b7cda33 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIconProvider.cs +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIconProvider.cs @@ -14,6 +14,7 @@ using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort.Joinables; using Rdmp.Core.Curation.Data.Dashboarding; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.Icons.IconOverlays; using Rdmp.Core.Icons.IconProvision.StateBasedIconProviders; @@ -193,9 +194,13 @@ protected virtual Image GetImageImpl(object concept, OverlayKind kind = } case IAtomicCommand cmd: return cmd.GetImage(this); + case Curation.Data.Datasets.Dataset: + case Curation.Data.Datasets.DatasetProviderConfiguration: + return GetImageImpl(RDMPConcept.Dataset); + case CatalogueDatasetLinkage: + return GetImageImpl(RDMPConcept.Catalogue); } - return ImageUnknown; } diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.Designer.cs b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.Designer.cs index fa1a2abc1e..10db07ba2f 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.Designer.cs +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.Designer.cs @@ -702,6 +702,15 @@ public static Byte[] CohortIdentificationConfiguration } } + + public static Byte[] TemplateCohortIdentificationConfiguration + { + get + { + object obj = ResourceManager.GetObject("TemplateCohortIdentificationConfiguration", resourceCulture); + return ((Byte[])(obj)); + } + } /// /// Looks up a localized resource of type Image. /// diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx index db264af676..6632602700 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.resx @@ -784,4 +784,22 @@ ..\StandardRegex31.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + ..\AllTemplateCohortIdentificationConfigurationsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\TemplateCohortIdentificationConfiguration.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AssociatedCohortIdentificationTemplatesNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\DataProvider.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\AllDatasetsNode.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\Dataset.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx index 7d3f1ac339..91db6c9af6 100644 --- a/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx +++ b/Rdmp.Core/Icons/IconProvision/CatalogueIcons.zh-Hans.resx @@ -121,4 +121,22 @@ ..\zh-Hans\Project.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs index b3988b6401..8e262cd2c6 100644 --- a/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs +++ b/Rdmp.Core/Icons/IconProvision/RDMPConcept.cs @@ -213,5 +213,10 @@ public enum RDMPConcept RegexRedaction, RegexRedactionConfiguration, RegexRedactionKey, - AllRegexRedactionConfigurationsNode + AllRegexRedactionConfigurationsNode, + DatasetProviderConfiguration, + AllDatasetProviderConfigurationsNode, + PluginDataset, + AllTemplateCohortIdentificationConfigurationsNode, + AssociatedCohortIdentificationTemplatesNode } \ No newline at end of file diff --git a/Rdmp.Core/Icons/IconProvision/StateBasedIconProviders/CohortIdentificationConfigurationStateBasedIconProvider.cs b/Rdmp.Core/Icons/IconProvision/StateBasedIconProviders/CohortIdentificationConfigurationStateBasedIconProvider.cs index 4729c49418..0703ceb6ba 100644 --- a/Rdmp.Core/Icons/IconProvision/StateBasedIconProviders/CohortIdentificationConfigurationStateBasedIconProvider.cs +++ b/Rdmp.Core/Icons/IconProvision/StateBasedIconProviders/CohortIdentificationConfigurationStateBasedIconProvider.cs @@ -18,6 +18,7 @@ public class CohortIdentificationConfigurationStateBasedIconProvider : IObjectSt private readonly Image _cohortIdentificationConfigurationVersion; private readonly Image _frozenCohortIdentificationConfiguration; private readonly Image _frozenCohortIdentificationConfigurationVersion; + private readonly Image _templateCogortIdentificationConfiguration; public CohortIdentificationConfigurationStateBasedIconProvider() { @@ -26,13 +27,14 @@ public CohortIdentificationConfigurationStateBasedIconProvider() _frozenCohortIdentificationConfiguration = Image.Load(CatalogueIcons.FrozenCohortIdentificationConfiguration); _frozenCohortIdentificationConfigurationVersion = IconOverlayProvider.GetOverlay(_frozenCohortIdentificationConfiguration, OverlayKind.Version); + _templateCogortIdentificationConfiguration = Image.Load(CatalogueIcons.TemplateCohortIdentificationConfiguration); } public Image GetImageIfSupportedObject(object o) => o is not CohortIdentificationConfiguration cic ? null - : cic.Frozen + : cic.IsTemplate?_templateCogortIdentificationConfiguration:cic.Frozen ? cic.Version is null?_frozenCohortIdentificationConfiguration: _frozenCohortIdentificationConfigurationVersion : cic.Version is null ? _cohortIdentificationConfiguration: _cohortIdentificationConfigurationVersion; } \ No newline at end of file diff --git a/Rdmp.Core/Icons/TemplateCohortIdentificationConfiguration.png b/Rdmp.Core/Icons/TemplateCohortIdentificationConfiguration.png new file mode 100644 index 0000000000..c263543476 Binary files /dev/null and b/Rdmp.Core/Icons/TemplateCohortIdentificationConfiguration.png differ diff --git a/Rdmp.Core/Providers/CatalogueChildProvider.cs b/Rdmp.Core/Providers/CatalogueChildProvider.cs index 16ae2f55ee..cd32e7a2a8 100644 --- a/Rdmp.Core/Providers/CatalogueChildProvider.cs +++ b/Rdmp.Core/Providers/CatalogueChildProvider.cs @@ -5,6 +5,7 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using System; +using System.ClientModel.Primitives; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; @@ -17,6 +18,7 @@ using Rdmp.Core.Curation.Data.Cohort.Joinables; using Rdmp.Core.Curation.Data.Dashboarding; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Pipelines; @@ -53,1916 +55,1941 @@ namespace Rdmp.Core.Providers; /// public class CatalogueChildProvider : ICoreChildProvider { - //Load System - public LoadMetadata[] AllLoadMetadatas { get; set; } + //Load System + public LoadMetadata[] AllLoadMetadatas { get; set; } + public LoadMetadataCatalogueLinkage[] AllLoadMetadataCatalogueLinkages { get; set; } - private LoadMetadataCatalogueLinkage[] AllLoadMetadataLinkage { get; set; } - public ProcessTask[] AllProcessTasks { get; set; } - public ProcessTaskArgument[] AllProcessTasksArguments { get; set; } + private LoadMetadataCatalogueLinkage[] AllLoadMetadataLinkage { get; set; } + public ProcessTask[] AllProcessTasks { get; set; } + public ProcessTaskArgument[] AllProcessTasksArguments { get; set; } - public LoadProgress[] AllLoadProgresses { get; set; } - public CacheProgress[] AllCacheProgresses { get; set; } - public PermissionWindow[] AllPermissionWindows { get; set; } + public LoadProgress[] AllLoadProgresses { get; set; } + public CacheProgress[] AllCacheProgresses { get; set; } + public PermissionWindow[] AllPermissionWindows { get; set; } //Catalogue side of things public Catalogue[] AllCatalogues { get; set; } - public Curation.Data.Dataset[] AllDatasets { get; set; } + public Curation.Data.Datasets.Dataset[] AllDatasets { get; set; } public Dictionary AllCataloguesDictionary { get; private set; } - public SupportingDocument[] AllSupportingDocuments { get; set; } - public SupportingSQLTable[] AllSupportingSQL { get; set; } + public SupportingDocument[] AllSupportingDocuments { get; set; } + public SupportingSQLTable[] AllSupportingSQL { get; set; } - //tells you the immediate children of a given node. Do not add to this directly instead add using AddToDictionaries unless you want the Key to be an 'on the sly' no known descendency child - private ConcurrentDictionary> _childDictionary = new(); + //tells you the immediate children of a given node. Do not add to this directly instead add using AddToDictionaries unless you want the Key to be an 'on the sly' no known descendency child + private ConcurrentDictionary> _childDictionary = new(); - //This is the reverse of _childDictionary in some ways. _childDictionary tells you the immediate children while - //this tells you for a given child object what the navigation tree down to get to it is e.g. ascendancy[child] would return [root,grandParent,parent] - private ConcurrentDictionary _descendancyDictionary = new(); + //This is the reverse of _childDictionary in some ways. _childDictionary tells you the immediate children while + //this tells you for a given child object what the navigation tree down to get to it is e.g. ascendancy[child] would return [root,grandParent,parent] + private ConcurrentDictionary _descendancyDictionary = new(); - public IEnumerable AllCatalogueItems => AllCatalogueItemsDictionary.Values; + public IEnumerable AllCatalogueItems => AllCatalogueItemsDictionary.Values; - private Dictionary> _catalogueToCatalogueItems; - public Dictionary AllCatalogueItemsDictionary { get; private set; } + private Dictionary> _catalogueToCatalogueItems; + public Dictionary AllCatalogueItemsDictionary { get; private set; } - private Dictionary _allColumnInfos; + private Dictionary _allColumnInfos; - public AggregateConfiguration[] AllAggregateConfigurations { get; private set; } - public AggregateDimension[] AllAggregateDimensions { get; private set; } + public AggregateConfiguration[] AllAggregateConfigurations { get; private set; } + public AggregateDimension[] AllAggregateDimensions { get; private set; } - public AggregateContinuousDateAxis[] AllAggregateContinuousDateAxis { get; private set; } + public AggregateContinuousDateAxis[] AllAggregateContinuousDateAxis { get; private set; } - public AllRDMPRemotesNode AllRDMPRemotesNode { get; private set; } - public RemoteRDMP[] AllRemoteRDMPs { get; set; } + public AllRDMPRemotesNode AllRDMPRemotesNode { get; private set; } + public RemoteRDMP[] AllRemoteRDMPs { get; set; } - public AllDashboardsNode AllDashboardsNode { get; set; } - public DashboardLayout[] AllDashboards { get; set; } + public AllDashboardsNode AllDashboardsNode { get; set; } + public DashboardLayout[] AllDashboards { get; set; } - public AllObjectSharingNode AllObjectSharingNode { get; private set; } - public ObjectImport[] AllImports { get; set; } - public ObjectExport[] AllExports { get; set; } + public AllObjectSharingNode AllObjectSharingNode { get; private set; } + public ObjectImport[] AllImports { get; set; } + public ObjectExport[] AllExports { get; set; } - public AllStandardRegexesNode AllStandardRegexesNode { get; private set; } - public AllPipelinesNode AllPipelinesNode { get; private set; } - public OtherPipelinesNode OtherPipelinesNode { get; private set; } - public Pipeline[] AllPipelines { get; set; } - public PipelineComponent[] AllPipelineComponents { get; set; } + public AllStandardRegexesNode AllStandardRegexesNode { get; private set; } + public AllPipelinesNode AllPipelinesNode { get; private set; } + public OtherPipelinesNode OtherPipelinesNode { get; private set; } + public Pipeline[] AllPipelines { get; set; } + public PipelineComponent[] AllPipelineComponents { get; set; } - public PipelineComponentArgument[] AllPipelineComponentsArguments { get; set; } + public PipelineComponentArgument[] AllPipelineComponentsArguments { get; set; } - public StandardRegex[] AllStandardRegexes { get; set; } + public StandardRegex[] AllStandardRegexes { get; set; } - //TableInfo side of things - public AllANOTablesNode AllANOTablesNode { get; private set; } - public ANOTable[] AllANOTables { get; set; } + //TableInfo side of things + public AllANOTablesNode AllANOTablesNode { get; private set; } + public ANOTable[] AllANOTables { get; set; } - public ExternalDatabaseServer[] AllExternalServers { get; private set; } - public TableInfoServerNode[] AllServers { get; private set; } - public TableInfo[] AllTableInfos { get; private set; } + public ExternalDatabaseServer[] AllExternalServers { get; private set; } + public TableInfoServerNode[] AllServers { get; private set; } + public TableInfo[] AllTableInfos { get; private set; } - public AllDataAccessCredentialsNode AllDataAccessCredentialsNode { get; set; } + public AllDataAccessCredentialsNode AllDataAccessCredentialsNode { get; set; } - public AllExternalServersNode AllExternalServersNode { get; private set; } - public AllServersNode AllServersNode { get; private set; } + public AllExternalServersNode AllExternalServersNode { get; private set; } + public AllServersNode AllServersNode { get; private set; } - public DataAccessCredentials[] AllDataAccessCredentials { get; set; } - public Dictionary> AllDataAccessCredentialUsages { get; set; } + public DataAccessCredentials[] AllDataAccessCredentials { get; set; } + public Dictionary> AllDataAccessCredentialUsages { get; set; } - public Dictionary> TableInfosToColumnInfos { get; private set; } - public ColumnInfo[] AllColumnInfos { get; private set; } - public PreLoadDiscardedColumn[] AllPreLoadDiscardedColumns { get; private set; } + public Dictionary> TableInfosToColumnInfos { get; private set; } + public ColumnInfo[] AllColumnInfos { get; private set; } + public PreLoadDiscardedColumn[] AllPreLoadDiscardedColumns { get; private set; } - public Lookup[] AllLookups { get; set; } + public Lookup[] AllLookups { get; set; } - public JoinInfo[] AllJoinInfos { get; set; } + public JoinInfo[] AllJoinInfos { get; set; } - public AnyTableSqlParameter[] AllAnyTableParameters; + public AnyTableSqlParameter[] AllAnyTableParameters; - //Filter / extraction side of things - public IEnumerable AllExtractionInformations => AllExtractionInformationsDictionary.Values; + //Filter / extraction side of things + public IEnumerable AllExtractionInformations => AllExtractionInformationsDictionary.Values; - public AllPermissionWindowsNode AllPermissionWindowsNode { get; set; } - public FolderNode LoadMetadataRootFolder { get; set; } + public AllPermissionWindowsNode AllPermissionWindowsNode { get; set; } + public FolderNode LoadMetadataRootFolder { get; set; } - public FolderNode DatasetRootFolder { get; set; } - public FolderNode CohortIdentificationConfigurationRootFolder { get; set; } - public FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; set; } + public FolderNode CohortIdentificationConfigurationRootFolder { get; set; } + public AllTemplateCohortIdentificationConfigurationsNode AllTemplateCohortIdentificationConfigurationsNode { get; set; } + public FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; set; } + public FolderNode DatasetRootFolder { get; set; } - public AllConnectionStringKeywordsNode AllConnectionStringKeywordsNode { get; set; } - public ConnectionStringKeyword[] AllConnectionStringKeywords { get; set; } + public AllConnectionStringKeywordsNode AllConnectionStringKeywordsNode { get; set; } + public ConnectionStringKeyword[] AllConnectionStringKeywords { get; set; } - public Dictionary AllExtractionInformationsDictionary { get; private set; } - protected Dictionary _extractionInformationsByCatalogueItem; + public Dictionary AllExtractionInformationsDictionary { get; private set; } + protected Dictionary _extractionInformationsByCatalogueItem; - private IFilterManager _aggregateFilterManager; + private IFilterManager _aggregateFilterManager; - //Filters for Aggregates (includes filter containers (AND/OR) - public Dictionary AllAggregateContainersDictionary { get; private set; } - public AggregateFilterContainer[] AllAggregateContainers => AllAggregateContainersDictionary.Values.ToArray(); + //Filters for Aggregates (includes filter containers (AND/OR) + public Dictionary AllAggregateContainersDictionary { get; private set; } + public AggregateFilterContainer[] AllAggregateContainers => AllAggregateContainersDictionary.Values.ToArray(); - public AggregateFilter[] AllAggregateFilters { get; private set; } - public AggregateFilterParameter[] AllAggregateFilterParameters { get; private set; } + public AggregateFilter[] AllAggregateFilters { get; private set; } + public AggregateFilterParameter[] AllAggregateFilterParameters { get; private set; } - //Catalogue master filters (does not include any support for filter containers (AND/OR) - private ExtractionFilter[] AllCatalogueFilters; - public ExtractionFilterParameter[] AllCatalogueParameters; - public ExtractionFilterParameterSet[] AllCatalogueValueSets; - public ExtractionFilterParameterSetValue[] AllCatalogueValueSetValues; + //Catalogue master filters (does not include any support for filter containers (AND/OR) + private ExtractionFilter[] AllCatalogueFilters; + public ExtractionFilterParameter[] AllCatalogueParameters; + public ExtractionFilterParameterSet[] AllCatalogueValueSets; + public ExtractionFilterParameterSetValue[] AllCatalogueValueSetValues; - private ICohortContainerManager _cohortContainerManager; + private ICohortContainerManager _cohortContainerManager; - public CohortIdentificationConfiguration[] AllCohortIdentificationConfigurations { get; private set; } - public CohortAggregateContainer[] AllCohortAggregateContainers { get; set; } - public JoinableCohortAggregateConfiguration[] AllJoinables { get; set; } - public JoinableCohortAggregateConfigurationUse[] AllJoinUses { get; set; } + public CohortIdentificationConfiguration[] AllCohortIdentificationConfigurations { get; private set; } + public CohortIdentificationConfiguration[] AllTemplateCohortIdentificationConfigurations { get; private set; } + public CohortAggregateContainer[] AllCohortAggregateContainers { get; set; } + public JoinableCohortAggregateConfiguration[] AllJoinables { get; set; } + public JoinableCohortAggregateConfigurationUse[] AllJoinUses { get; set; } - /// - /// Collection of all objects for which there are masqueraders - /// - public ConcurrentDictionary> AllMasqueraders { get; private set; } + /// + /// Collection of all objects for which there are masqueraders + /// + public ConcurrentDictionary> AllMasqueraders { get; private set; } - private IChildProvider[] _pluginChildProviders; - private readonly ICatalogueRepository _catalogueRepository; - private readonly ICheckNotifier _errorsCheckNotifier; - private readonly List _blockedPlugins = new(); + private IChildProvider[] _pluginChildProviders; + private readonly ICatalogueRepository _catalogueRepository; + private readonly ICheckNotifier _errorsCheckNotifier; + private readonly List _blockedPlugins = new(); - public AllGovernanceNode AllGovernanceNode { get; private set; } - public GovernancePeriod[] AllGovernancePeriods { get; private set; } - public GovernanceDocument[] AllGovernanceDocuments { get; private set; } - public Dictionary> GovernanceCoverage { get; private set; } + public AllGovernanceNode AllGovernanceNode { get; private set; } + public GovernancePeriod[] AllGovernancePeriods { get; private set; } + public GovernanceDocument[] AllGovernanceDocuments { get; private set; } + public Dictionary> GovernanceCoverage { get; private set; } - private CommentStore _commentStore; + private CommentStore _commentStore; - public JoinableCohortAggregateConfigurationUse[] AllJoinableCohortAggregateConfigurationUse { get; private set; } - public AllPluginsNode AllPluginsNode { get; private set; } - public HashSet PipelineUseCases { get; set; } = new(); + public JoinableCohortAggregateConfigurationUse[] AllJoinableCohortAggregateConfigurationUse { get; private set; } + public AllPluginsNode AllPluginsNode { get; private set; } + public HashSet PipelineUseCases { get; set; } = new(); - /// - /// Lock for changes to Child provider - /// - protected object WriteLock = new(); + /// + /// Lock for changes to Child provider + /// + protected object WriteLock = new(); - public AllOrphanAggregateConfigurationsNode OrphanAggregateConfigurationsNode { get; set; } = new(); - public AllTemplateAggregateConfigurationsNode TemplateAggregateConfigurationsNode { get; set; } = new(); - public FolderNode CatalogueRootFolder { get; private set; } + public AllOrphanAggregateConfigurationsNode OrphanAggregateConfigurationsNode { get; set; } = new(); + public AllTemplateAggregateConfigurationsNode TemplateAggregateConfigurationsNode { get; set; } = new(); + public FolderNode CatalogueRootFolder { get; private set; } - public AllDatasetsNode AllDatasetsNode { get; set; } + public AllDatasetsNode AllDatasetsNode { get; set; } public RegexRedactionConfiguration[] AllRegexRedactionConfigurations { get; set; } public AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; set; } + public AllDatasetProviderConfigurationsNode AllDatasetProviderConfigurationsNode { get; set; } + public DatasetProviderConfiguration[] DatasetProviderConfigurations { get; set; } - public HashSet OrphanAggregateConfigurations; + public HashSet OrphanAggregateConfigurations; public AggregateConfiguration[] TemplateAggregateConfigurations; - protected Stopwatch ProgressStopwatch = Stopwatch.StartNew(); - private int _progress; + protected Stopwatch ProgressStopwatch = Stopwatch.StartNew(); + private int _progress; - /// - /// - /// - /// - /// - /// Where to report errors building the hierarchy e.g. when crash. Set to null for - /// Previous child provider state if you know it otherwise null - public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] pluginChildProviders, - ICheckNotifier errorsCheckNotifier, CatalogueChildProvider previousStateIfKnown) - { - _commentStore = repository.CommentStore; - _catalogueRepository = repository; - _catalogueRepository?.EncryptionManager?.ClearAllInjections(); - - _errorsCheckNotifier = errorsCheckNotifier ?? IgnoreAllErrorsCheckNotifier.Instance; - - if (UserSettings.DebugPerformance) - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( - $"Refresh generated by:{Environment.NewLine}{Environment.StackTrace}", CheckResult.Success)); - - // all the objects which are - AllMasqueraders = new ConcurrentDictionary>(); - - _pluginChildProviders = pluginChildProviders ?? Array.Empty(); - - ReportProgress("Before object fetches"); - - AllAnyTableParameters = GetAllObjects(repository); - - AllANOTables = GetAllObjects(repository); - AllANOTablesNode = new AllANOTablesNode(); - AddChildren(AllANOTablesNode); - - AllCatalogues = GetAllObjects(repository); - AllCataloguesDictionary = AllCatalogues.ToDictionaryEx(i => i.ID, o => o); - - AllDatasets = GetAllObjects(repository); - - AllLoadMetadatas = GetAllObjects(repository); - AllLoadMetadataLinkage = GetAllObjects(repository); - AllProcessTasks = GetAllObjects(repository); - AllProcessTasksArguments = GetAllObjects(repository); - AllLoadProgresses = GetAllObjects(repository); - AllCacheProgresses = GetAllObjects(repository); - - AllPermissionWindows = GetAllObjects(repository); - AllPermissionWindowsNode = new AllPermissionWindowsNode(); - AddChildren(AllPermissionWindowsNode); - - AllRemoteRDMPs = GetAllObjects(repository); - - AllExternalServers = GetAllObjects(repository); - - AllTableInfos = GetAllObjects(repository); - AllDataAccessCredentials = GetAllObjects(repository); - AllDataAccessCredentialsNode = new AllDataAccessCredentialsNode(); - AddChildren(AllDataAccessCredentialsNode); - - AllConnectionStringKeywordsNode = new AllConnectionStringKeywordsNode(); - AllConnectionStringKeywords = GetAllObjects(repository).ToArray(); - AddToDictionaries(new HashSet(AllConnectionStringKeywords), - new DescendancyList(AllConnectionStringKeywordsNode)); - - ReportProgress("after basic object fetches"); - - Task.WaitAll( - //which TableInfos use which Credentials under which DataAccessContexts - Task.Factory.StartNew(() => - { - AllDataAccessCredentialUsages = - repository.TableInfoCredentialsManager.GetAllCredentialUsagesBy(AllDataAccessCredentials, - AllTableInfos); - }), - Task.Factory.StartNew(() => { AllColumnInfos = GetAllObjects(repository); }) - ); - - ReportProgress("After credentials"); - - TableInfosToColumnInfos = AllColumnInfos.GroupBy(c => c.TableInfo_ID) - .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); - - ReportProgress("After TableInfo to ColumnInfo mapping"); + /// + /// + /// + /// + /// + /// Where to report errors building the hierarchy e.g. when crash. Set to null for + /// Previous child provider state if you know it otherwise null + public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[] pluginChildProviders, + ICheckNotifier errorsCheckNotifier, CatalogueChildProvider previousStateIfKnown) + { + _commentStore = repository.CommentStore; + _catalogueRepository = repository; + _catalogueRepository?.EncryptionManager?.ClearAllInjections(); - AllPreLoadDiscardedColumns = GetAllObjects(repository); + _errorsCheckNotifier = errorsCheckNotifier ?? IgnoreAllErrorsCheckNotifier.Instance; - AllSupportingDocuments = GetAllObjects(repository); - AllSupportingSQL = GetAllObjects(repository); + if (UserSettings.DebugPerformance) + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( + $"Refresh generated by:{Environment.NewLine}{Environment.StackTrace}", CheckResult.Success)); - AllCohortIdentificationConfigurations = GetAllObjects(repository); + // all the objects which are + AllMasqueraders = new ConcurrentDictionary>(); - FetchCatalogueItems(); + _pluginChildProviders = pluginChildProviders ?? Array.Empty(); - ReportProgress("After CatalogueItem injection"); + ReportProgress("Before object fetches"); - FetchExtractionInformations(); + AllAnyTableParameters = GetAllObjects(repository); - ReportProgress("After ExtractionInformation injection"); + AllANOTables = GetAllObjects(repository); + AllANOTablesNode = new AllANOTablesNode(); + AddChildren(AllANOTablesNode); - BuildAggregateConfigurations(); + AllCatalogues = GetAllObjects(repository); + AllCataloguesDictionary = AllCatalogues.ToDictionaryEx(i => i.ID, o => o); - BuildCohortCohortAggregateContainers(); + AllDatasets = GetAllObjects(repository); - AllJoinables = GetAllObjects(repository); - AllJoinUses = GetAllObjects(repository); + AllLoadMetadatas = GetAllObjects(repository); + AllLoadMetadataCatalogueLinkages = GetAllObjects(repository); + AllLoadMetadataLinkage = GetAllObjects(repository); + AllProcessTasks = GetAllObjects(repository); + AllProcessTasksArguments = GetAllObjects(repository); + AllLoadProgresses = GetAllObjects(repository); + AllCacheProgresses = GetAllObjects(repository); - AllCatalogueFilters = GetAllObjects(repository); - AllCatalogueParameters = GetAllObjects(repository); - AllCatalogueValueSets = GetAllObjects(repository); - AllCatalogueValueSetValues = GetAllObjects(repository); + AllPermissionWindows = GetAllObjects(repository); + AllPermissionWindowsNode = new AllPermissionWindowsNode(); + AddChildren(AllPermissionWindowsNode); - ReportProgress("After Filter and Joinable fetching"); + AllRemoteRDMPs = GetAllObjects(repository); + AllExternalServers = GetAllObjects(repository); - AllLookups = GetAllObjects(repository); + AllTableInfos = GetAllObjects(repository); + AllDataAccessCredentials = GetAllObjects(repository); + AllDataAccessCredentialsNode = new AllDataAccessCredentialsNode(); + AddChildren(AllDataAccessCredentialsNode); - foreach (var l in AllLookups) - l.SetKnownColumns(_allColumnInfos[l.PrimaryKey_ID], _allColumnInfos[l.ForeignKey_ID], - _allColumnInfos[l.Description_ID]); + AllConnectionStringKeywordsNode = new AllConnectionStringKeywordsNode(); + AllConnectionStringKeywords = GetAllObjects(repository).ToArray(); + AddToDictionaries(new HashSet(AllConnectionStringKeywords), + new DescendancyList(AllConnectionStringKeywordsNode)); - AllJoinInfos = repository.GetAllObjects(); + ReportProgress("after basic object fetches"); - foreach (var j in AllJoinInfos) - j.SetKnownColumns(_allColumnInfos[j.PrimaryKey_ID], _allColumnInfos[j.ForeignKey_ID]); + Task.WaitAll( + //which TableInfos use which Credentials under which DataAccessContexts + Task.Factory.StartNew(() => + { + AllDataAccessCredentialUsages = + repository.TableInfoCredentialsManager.GetAllCredentialUsagesBy(AllDataAccessCredentials, + AllTableInfos); + }), + Task.Factory.StartNew(() => { AllColumnInfos = GetAllObjects(repository); }) + ); - ReportProgress("After SetKnownColumns"); + ReportProgress("After credentials"); - AllExternalServersNode = new AllExternalServersNode(); - AddChildren(AllExternalServersNode); + TableInfosToColumnInfos = AllColumnInfos.GroupBy(c => c.TableInfo_ID) + .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); - AllRDMPRemotesNode = new AllRDMPRemotesNode(); - AddChildren(AllRDMPRemotesNode); + ReportProgress("After TableInfo to ColumnInfo mapping"); - AllDashboardsNode = new AllDashboardsNode(); - AllDashboards = GetAllObjects(repository); - AddChildren(AllDashboardsNode); + AllPreLoadDiscardedColumns = GetAllObjects(repository); - AllObjectSharingNode = new AllObjectSharingNode(); - AllExports = GetAllObjects(repository); - AllImports = GetAllObjects(repository); + AllSupportingDocuments = GetAllObjects(repository); + AllSupportingSQL = GetAllObjects(repository); - AddChildren(AllObjectSharingNode); + AllCohortIdentificationConfigurations = GetAllObjects(repository).Where(cic => !cic.IsTemplate).ToArray(); + AllTemplateCohortIdentificationConfigurations = GetAllObjects(repository).Where(cic => cic.IsTemplate).ToArray(); - ReportProgress("After Object Sharing discovery"); + FetchCatalogueItems(); - //Pipelines setup (see also DataExportChildProvider for calls to AddPipelineUseCases) - //Root node for all pipelines - AllPipelinesNode = new AllPipelinesNode(); + ReportProgress("After CatalogueItem injection"); - //Pipelines not found to be part of any use case after AddPipelineUseCases - OtherPipelinesNode = new OtherPipelinesNode(); - AllPipelines = GetAllObjects(repository); - AllPipelineComponents = GetAllObjects(repository); - AllPipelineComponentsArguments = GetAllObjects(repository); + FetchExtractionInformations(); - foreach (var p in AllPipelines) - p.InjectKnown(AllPipelineComponents.Where(pc => pc.Pipeline_ID == p.ID).ToArray()); + ReportProgress("After ExtractionInformation injection"); - AllStandardRegexesNode = new AllStandardRegexesNode(); - AllStandardRegexes = GetAllObjects(repository); - AddToDictionaries(new HashSet(AllStandardRegexes), new DescendancyList(AllStandardRegexesNode)); + BuildAggregateConfigurations(); - ReportProgress("After Pipelines setup"); + BuildCohortCohortAggregateContainers(); - //All the things for TableInfoCollectionUI - BuildServerNodes(); + AllJoinables = GetAllObjects(repository); + AllJoinUses = GetAllObjects(repository); - ReportProgress("BuildServerNodes"); + AllCatalogueFilters = GetAllObjects(repository); + AllCatalogueParameters = GetAllObjects(repository); + AllCatalogueValueSets = GetAllObjects(repository); + AllCatalogueValueSetValues = GetAllObjects(repository); - //add a new CatalogueItemNodes - InjectCatalogueItems(); + ReportProgress("After Filter and Joinable fetching"); - CatalogueRootFolder = FolderHelper.BuildFolderTree(AllCatalogues); - AddChildren(CatalogueRootFolder, new DescendancyList(CatalogueRootFolder)); + AllLookups = GetAllObjects(repository); - DatasetRootFolder = FolderHelper.BuildFolderTree(AllDatasets); - AddChildren(DatasetRootFolder, new DescendancyList(DatasetRootFolder)); + foreach (var l in AllLookups) + l.SetKnownColumns(_allColumnInfos[l.PrimaryKey_ID], _allColumnInfos[l.ForeignKey_ID], + _allColumnInfos[l.Description_ID]); - ReportProgress("Build Catalogue Folder Root"); + AllJoinInfos = repository.GetAllObjects(); - LoadMetadataRootFolder = FolderHelper.BuildFolderTree(AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID is null).ToArray()); - AddChildren(LoadMetadataRootFolder, new DescendancyList(LoadMetadataRootFolder)); + foreach (var j in AllJoinInfos) + j.SetKnownColumns(_allColumnInfos[j.PrimaryKey_ID], _allColumnInfos[j.ForeignKey_ID]); - CohortIdentificationConfigurationRootFolder = - FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations); - AddChildren(CohortIdentificationConfigurationRootFolder, - new DescendancyList(CohortIdentificationConfigurationRootFolder)); + ReportProgress("After SetKnownColumns"); - CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations = FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations.Where(cic => cic.Version is null).ToArray()); - AddChildren(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations, - new DescendancyList(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations)); - var templateAggregateConfigurationIds = - new HashSet( - repository.GetExtendedProperties(ExtendedProperty.IsTemplate) - .Where(p => p.ReferencedObjectType.Equals(nameof(AggregateConfiguration))) - .Select(r => r.ReferencedObjectID)); + AllExternalServersNode = new AllExternalServersNode(); + AddChildren(AllExternalServersNode); - TemplateAggregateConfigurations = AllAggregateConfigurations - .Where(ac => templateAggregateConfigurationIds.Contains(ac.ID)).ToArray(); + AllRDMPRemotesNode = new AllRDMPRemotesNode(); + AddChildren(AllRDMPRemotesNode); - //add the orphans under the orphan folder - AddToDictionaries(new HashSet(OrphanAggregateConfigurations), - new DescendancyList(OrphanAggregateConfigurationsNode)); + AllDashboardsNode = new AllDashboardsNode(); + AllDashboards = GetAllObjects(repository); + AddChildren(AllDashboardsNode); - var dec = new DescendancyList(TemplateAggregateConfigurationsNode); - dec.SetBetterRouteExists(); - AddToDictionaries(new HashSet(TemplateAggregateConfigurations), dec); + AllObjectSharingNode = new AllObjectSharingNode(); + AllExports = GetAllObjects(repository); + AllImports = GetAllObjects(repository); - //Some AggregateConfigurations are 'Patient Index Tables', this happens when there is an existing JoinableCohortAggregateConfiguration declared where - //the AggregateConfiguration_ID is the AggregateConfiguration.ID. We can inject this knowledge now so to avoid database lookups later (e.g. at icon provision time) - var joinableDictionaryByAggregateConfigurationId = - AllJoinables.ToDictionaryEx(j => j.AggregateConfiguration_ID, v => v); + AddChildren(AllObjectSharingNode); - foreach (var ac in AllAggregateConfigurations) //if there's a joinable - ac.InjectKnown( //inject that we know the joinable (and what it is) - joinableDictionaryByAggregateConfigurationId.GetValueOrDefault(ac.ID)); //otherwise inject that it is not a joinable (suppresses database checking later) + ReportProgress("After Object Sharing discovery"); - ReportProgress("After AggregateConfiguration injection"); + //Pipelines setup (see also DataExportChildProvider for calls to AddPipelineUseCases) + //Root node for all pipelines + AllPipelinesNode = new AllPipelinesNode(); - AllGovernanceNode = new AllGovernanceNode(); - AllGovernancePeriods = GetAllObjects(repository); - AllGovernanceDocuments = GetAllObjects(repository); - GovernanceCoverage = repository.GovernanceManager.GetAllGovernedCataloguesForAllGovernancePeriods(); + //Pipelines not found to be part of any use case after AddPipelineUseCases + OtherPipelinesNode = new OtherPipelinesNode(); + AllPipelines = GetAllObjects(repository); + AllPipelineComponents = GetAllObjects(repository); + AllPipelineComponentsArguments = GetAllObjects(repository); - AddChildren(AllGovernanceNode); + foreach (var p in AllPipelines) + p.InjectKnown(AllPipelineComponents.Where(pc => pc.Pipeline_ID == p.ID).ToArray()); - ReportProgress("After Governance"); + AllStandardRegexesNode = new AllStandardRegexesNode(); + AllStandardRegexes = GetAllObjects(repository); + AddToDictionaries(new HashSet(AllStandardRegexes), new DescendancyList(AllStandardRegexesNode)); - AllPluginsNode = new AllPluginsNode(); - AddChildren(AllPluginsNode); + ReportProgress("After Pipelines setup"); - ReportProgress("After Plugins"); + //All the things for TableInfoCollectionUI + BuildServerNodes(); - AllRegexRedactionConfigurations = GetAllObjects(repository); - AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); - AddChildren(AllRegexRedactionConfigurationsNode); + ReportProgress("BuildServerNodes"); + //add a new CatalogueItemNodes + InjectCatalogueItems(); - AllDatasets = GetAllObjects(repository); - AllDatasetsNode = new AllDatasetsNode(); - AddChildren(AllDatasetsNode); + CatalogueRootFolder = FolderHelper.BuildFolderTree(AllCatalogues); + AddChildren(CatalogueRootFolder, new DescendancyList(CatalogueRootFolder)); - ReportProgress("After Configurations"); - var searchables = new Dictionary>(); + DatasetRootFolder = FolderHelper.BuildFolderTree(AllDatasets); + AddChildren(DatasetRootFolder, new DescendancyList(DatasetRootFolder)); - foreach (var o in _descendancyDictionary.Keys.OfType()) - { - if (!searchables.ContainsKey(o.ID)) - searchables.Add(o.ID, new HashSet()); + ReportProgress("Build Catalogue Folder Root"); - searchables[o.ID].Add(o); - } + LoadMetadataRootFolder = FolderHelper.BuildFolderTree(AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID is null).ToArray()); + AddChildren(LoadMetadataRootFolder, new DescendancyList(LoadMetadataRootFolder)); - ReportProgress("After building Searchables"); + CohortIdentificationConfigurationRootFolder = + FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations); + AddChildren(CohortIdentificationConfigurationRootFolder, + new DescendancyList(CohortIdentificationConfigurationRootFolder)); - foreach (var e in AllExports) - { - if (!searchables.TryGetValue(e.ReferencedObjectID, out var searchable)) - continue; + CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations = FolderHelper.BuildFolderTree(AllCohortIdentificationConfigurations.Where(cic => cic.Version is null).ToArray()); + AddChildren(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations, + new DescendancyList(CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations)); - var known = searchable - .FirstOrDefault(s => e.ReferencedObjectType == s.GetType().FullName); + var templateAggregateConfigurationIds = + new HashSet( + repository.GetExtendedProperties(ExtendedProperty.IsTemplate) + .Where(p => p.ReferencedObjectType.Equals(nameof(AggregateConfiguration))) + .Select(r => r.ReferencedObjectID)); - if (known != null) - e.InjectKnown(known); - } + AllTemplateCohortIdentificationConfigurationsNode = new AllTemplateCohortIdentificationConfigurationsNode(); + var templateCICTree = FolderHelper.BuildFolderTree(AllTemplateCohortIdentificationConfigurations); + AddChildren(templateCICTree, new DescendancyList(AllTemplateCohortIdentificationConfigurationsNode)); - ReportProgress("After building exports"); - } + TemplateAggregateConfigurations = AllAggregateConfigurations + .Where(ac => templateAggregateConfigurationIds.Contains(ac.ID)).ToArray(); + //add the orphans under the orphan folder + AddToDictionaries(new HashSet(OrphanAggregateConfigurations), + new DescendancyList(OrphanAggregateConfigurationsNode)); - private void FetchCatalogueItems() - { - AllCatalogueItemsDictionary = - GetAllObjects(_catalogueRepository).ToDictionaryEx(i => i.ID, o => o); + var dec = new DescendancyList(TemplateAggregateConfigurationsNode); + dec.SetBetterRouteExists(); + AddToDictionaries(new HashSet(TemplateAggregateConfigurations), dec); - ReportProgress("After CatalogueItem getting"); + //Some AggregateConfigurations are 'Patient Index Tables', this happens when there is an existing JoinableCohortAggregateConfiguration declared where + //the AggregateConfiguration_ID is the AggregateConfiguration.ID. We can inject this knowledge now so to avoid database lookups later (e.g. at icon provision time) + var joinableDictionaryByAggregateConfigurationId = + AllJoinables.ToDictionaryEx(j => j.AggregateConfiguration_ID, v => v); - _catalogueToCatalogueItems = AllCatalogueItems.GroupBy(c => c.Catalogue_ID) - .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); - _allColumnInfos = AllColumnInfos.ToDictionaryEx(i => i.ID, o => o); + foreach (var ac in AllAggregateConfigurations) //if there's a joinable + ac.InjectKnown( //inject that we know the joinable (and what it is) + joinableDictionaryByAggregateConfigurationId.GetValueOrDefault(ac.ID)); //otherwise inject that it is not a joinable (suppresses database checking later) - ReportProgress("After CatalogueItem Dictionary building"); + ReportProgress("After AggregateConfiguration injection"); - //Inject known ColumnInfos into CatalogueItems - Parallel.ForEach(AllCatalogueItems, ci => - { - if (ci.ColumnInfo_ID != null && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) - ci.InjectKnown(col); - else - ci.InjectKnown((ColumnInfo)null); - }); - } + AllGovernanceNode = new AllGovernanceNode(); + AllGovernancePeriods = GetAllObjects(repository); + AllGovernanceDocuments = GetAllObjects(repository); + GovernanceCoverage = repository.GovernanceManager.GetAllGovernedCataloguesForAllGovernancePeriods(); - private void FetchExtractionInformations() - { - AllExtractionInformationsDictionary = GetAllObjects(_catalogueRepository) - .ToDictionaryEx(i => i.ID, o => o); - _extractionInformationsByCatalogueItem = - AllExtractionInformationsDictionary.Values.ToDictionaryEx(k => k.CatalogueItem_ID, v => v); - - //Inject known CatalogueItems into ExtractionInformations - foreach (var ei in AllExtractionInformationsDictionary.Values) - if (AllCatalogueItemsDictionary.TryGetValue(ei.CatalogueItem_ID, out var ci)) - { - ei.InjectKnown(ci.ColumnInfo); - ei.InjectKnown(ci); - } - } + AddChildren(AllGovernanceNode); - private void BuildCohortCohortAggregateContainers() - { - AllCohortAggregateContainers = GetAllObjects(_catalogueRepository); + ReportProgress("After Governance"); + AllPluginsNode = new AllPluginsNode(); + AddChildren(AllPluginsNode); - //if we have a database repository then we should get answers from the caching version CohortContainerManagerFromChildProvider otherwise - //just use the one that is configured on the repository. + ReportProgress("After Plugins"); - _cohortContainerManager = _catalogueRepository is CatalogueRepository cataRepo - ? new CohortContainerManagerFromChildProvider(cataRepo, this) - : _catalogueRepository.CohortContainerManager; - } + AllRegexRedactionConfigurations = GetAllObjects(repository); + AllRegexRedactionConfigurationsNode = new AllRegexRedactionConfigurationsNode(); + AddChildren(AllRegexRedactionConfigurationsNode); - private void BuildAggregateConfigurations() - { - AllJoinableCohortAggregateConfigurationUse = - GetAllObjects(_catalogueRepository); - AllAggregateConfigurations = GetAllObjects(_catalogueRepository); + AllDatasetProviderConfigurationsNode = new AllDatasetProviderConfigurationsNode(); + DatasetProviderConfigurations = GetAllObjects(repository); + AddChildren(AllDatasetProviderConfigurationsNode); - BuildAggregateDimensions(); + AllDatasets = GetAllObjects(repository); + AllDatasetsNode = new AllDatasetsNode(); + AddChildren(AllDatasetsNode); - //to start with all aggregates are orphans (we prune this as we determine descendency in AddChildren methods - OrphanAggregateConfigurations = - new HashSet( - AllAggregateConfigurations.Where(ac => ac.IsCohortIdentificationAggregate)); + ReportProgress("After Configurations"); - foreach (var configuration in AllAggregateConfigurations) - { - configuration.InjectKnown(AllCataloguesDictionary[configuration.Catalogue_ID]); - configuration.InjectKnown(AllAggregateDimensions.Where(d => d.AggregateConfiguration_ID == configuration.ID) - .ToArray()); - } + var searchables = new Dictionary>(); - foreach (var d in AllAggregateDimensions) - d.InjectKnown(AllExtractionInformationsDictionary[d.ExtractionInformation_ID]); + foreach (var o in _descendancyDictionary.Keys.OfType()) + { + if (!searchables.ContainsKey(o.ID)) + searchables.Add(o.ID, new HashSet()); - ReportProgress("AggregateDimension injections"); + searchables[o.ID].Add(o); + } - BuildAggregateFilterContainers(); - } + ReportProgress("After building Searchables"); - private void BuildAggregateDimensions() - { - AllAggregateDimensions = GetAllObjects(_catalogueRepository); - AllAggregateContinuousDateAxis = GetAllObjects(_catalogueRepository); - } + foreach (var e in AllExports) + { + if (!searchables.TryGetValue(e.ReferencedObjectID, out var searchable)) + continue; - private void BuildAggregateFilterContainers() - { - AllAggregateContainersDictionary = GetAllObjects(_catalogueRepository) - .ToDictionaryEx(o => o.ID, o2 => o2); - AllAggregateFilters = GetAllObjects(_catalogueRepository); - AllAggregateFilterParameters = GetAllObjects(_catalogueRepository); - - _aggregateFilterManager = _catalogueRepository is CatalogueRepository cataRepo - ? new FilterManagerFromChildProvider(cataRepo, this) - : _catalogueRepository.FilterManager; - } + var known = searchable + .FirstOrDefault(s => e.ReferencedObjectType == s.GetType().FullName); + if (known != null) + e.InjectKnown(known); + } - protected void ReportProgress(string desc) - { - if (UserSettings.DebugPerformance) - { - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( - $"ChildProvider Stage {_progress++} ({desc}):{ProgressStopwatch.ElapsedMilliseconds}ms", - CheckResult.Success)); - ProgressStopwatch.Restart(); - } - } + ReportProgress("After building exports"); + } - private void AddChildren(AllPluginsNode allPluginsNode) - { - var children = new HashSet(LoadModuleAssembly.Assemblies); - var descendancy = new DescendancyList(allPluginsNode); - AddToDictionaries(children, descendancy); - } - private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) + private void FetchCatalogueItems() + { + AllCatalogueItemsDictionary = + GetAllObjects(_catalogueRepository).ToDictionaryEx(i => i.ID, o => o); + + ReportProgress("After CatalogueItem getting"); + + _catalogueToCatalogueItems = AllCatalogueItems.GroupBy(c => c.Catalogue_ID) + .ToDictionaryEx(gdc => gdc.Key, gdc => gdc.ToList()); + _allColumnInfos = AllColumnInfos.ToDictionaryEx(i => i.ID, o => o); + + ReportProgress("After CatalogueItem Dictionary building"); + + //Inject known ColumnInfos into CatalogueItems + Parallel.ForEach(AllCatalogueItems, ci => + { + if (ci.ColumnInfo_ID != null && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) + ci.InjectKnown(col); + else + ci.InjectKnown((ColumnInfo)null); + }); + } + + private void FetchExtractionInformations() + { + AllExtractionInformationsDictionary = GetAllObjects(_catalogueRepository) + .ToDictionaryEx(i => i.ID, o => o); + _extractionInformationsByCatalogueItem = + AllExtractionInformationsDictionary.Values.ToDictionaryEx(k => k.CatalogueItem_ID, v => v); + + //Inject known CatalogueItems into ExtractionInformations + foreach (var ei in AllExtractionInformationsDictionary.Values) + if (AllCatalogueItemsDictionary.TryGetValue(ei.CatalogueItem_ID, out var ci)) + { + ei.InjectKnown(ci.ColumnInfo); + ei.InjectKnown(ci); + } + } + + private void BuildCohortCohortAggregateContainers() + { + AllCohortAggregateContainers = GetAllObjects(_catalogueRepository); + + + //if we have a database repository then we should get answers from the caching version CohortContainerManagerFromChildProvider otherwise + //just use the one that is configured on the repository. + + _cohortContainerManager = _catalogueRepository is CatalogueRepository cataRepo + ? new CohortContainerManagerFromChildProvider(cataRepo, this) + : _catalogueRepository.CohortContainerManager; + } + + private void BuildAggregateConfigurations() + { + AllJoinableCohortAggregateConfigurationUse = + GetAllObjects(_catalogueRepository); + AllAggregateConfigurations = GetAllObjects(_catalogueRepository); + + BuildAggregateDimensions(); + + //to start with all aggregates are orphans (we prune this as we determine descendency in AddChildren methods + OrphanAggregateConfigurations = + new HashSet( + AllAggregateConfigurations.Where(ac => ac.IsCohortIdentificationAggregate)); + + foreach (var configuration in AllAggregateConfigurations) + { + configuration.InjectKnown(AllCataloguesDictionary[configuration.Catalogue_ID]); + configuration.InjectKnown(AllAggregateDimensions.Where(d => d.AggregateConfiguration_ID == configuration.ID) + .ToArray()); + } + + foreach (var d in AllAggregateDimensions) + d.InjectKnown(AllExtractionInformationsDictionary[d.ExtractionInformation_ID]); + + ReportProgress("AggregateDimension injections"); + + BuildAggregateFilterContainers(); + } + + private void BuildAggregateDimensions() + { + AllAggregateDimensions = GetAllObjects(_catalogueRepository); + AllAggregateContinuousDateAxis = GetAllObjects(_catalogueRepository); + } + + private void BuildAggregateFilterContainers() + { + AllAggregateContainersDictionary = GetAllObjects(_catalogueRepository) + .ToDictionaryEx(o => o.ID, o2 => o2); + AllAggregateFilters = GetAllObjects(_catalogueRepository); + AllAggregateFilterParameters = GetAllObjects(_catalogueRepository); + + _aggregateFilterManager = _catalogueRepository is CatalogueRepository cataRepo + ? new FilterManagerFromChildProvider(cataRepo, this) + : _catalogueRepository.FilterManager; + } + + + protected void ReportProgress(string desc) + { + if (UserSettings.DebugPerformance) + { + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( + $"ChildProvider Stage {_progress++} ({desc}):{ProgressStopwatch.ElapsedMilliseconds}ms", + CheckResult.Success)); + ProgressStopwatch.Restart(); + } + } + + private void AddChildren(AllPluginsNode allPluginsNode) + { + var children = new HashSet(LoadModuleAssembly.Assemblies); + var descendancy = new DescendancyList(allPluginsNode); + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllDatasetProviderConfigurationsNode allDatasetConfigurationNode) + { + var children = new HashSet(DatasetProviderConfigurations); + var descendancy = new DescendancyList(allDatasetConfigurationNode); + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllRegexRedactionConfigurationsNode allRegexRedactionConfigurationsNode) { var children = new HashSet(AllRegexRedactionConfigurations); var descendancy = new DescendancyList(allRegexRedactionConfigurationsNode); AddToDictionaries(children, descendancy); } - private void AddChildren(AllDatasetsNode allDatasetsNode) - { - var children = new HashSet(AllDatasets); - var descendancy = new DescendancyList(allDatasetsNode); - AddToDictionaries(children, descendancy); - } + private void AddChildren(AllDatasetsNode allDatasetsNode) + { + var children = new HashSet(AllDatasets); + var descendancy = new DescendancyList(allDatasetsNode); + AddToDictionaries(children, descendancy); + } - private void AddChildren(AllGovernanceNode allGovernanceNode) - { - var children = new HashSet(); - var descendancy = new DescendancyList(allGovernanceNode); + private void AddChildren(AllGovernanceNode allGovernanceNode) + { + var children = new HashSet(); + var descendancy = new DescendancyList(allGovernanceNode); - foreach (var gp in AllGovernancePeriods) - { - children.Add(gp); - AddChildren(gp, descendancy.Add(gp)); - } + foreach (var gp in AllGovernancePeriods) + { + children.Add(gp); + AddChildren(gp, descendancy.Add(gp)); + } - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void AddChildren(GovernancePeriod governancePeriod, DescendancyList descendancy) - { - var children = new HashSet(); + private void AddChildren(GovernancePeriod governancePeriod, DescendancyList descendancy) + { + var children = new HashSet(); - foreach (var doc in AllGovernanceDocuments.Where(d => d.GovernancePeriod_ID == governancePeriod.ID)) - children.Add(doc); + foreach (var doc in AllGovernanceDocuments.Where(d => d.GovernancePeriod_ID == governancePeriod.ID)) + children.Add(doc); - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void AddChildren(AllPermissionWindowsNode allPermissionWindowsNode) - { - var descendancy = new DescendancyList(allPermissionWindowsNode); + private void AddChildren(AllPermissionWindowsNode allPermissionWindowsNode) + { + var descendancy = new DescendancyList(allPermissionWindowsNode); - foreach (var permissionWindow in AllPermissionWindows) - AddChildren(permissionWindow, descendancy.Add(permissionWindow)); + foreach (var permissionWindow in AllPermissionWindows) + AddChildren(permissionWindow, descendancy.Add(permissionWindow)); - AddToDictionaries(new HashSet(AllPermissionWindows), descendancy); - } + AddToDictionaries(new HashSet(AllPermissionWindows), descendancy); + } - private void AddChildren(PermissionWindow permissionWindow, DescendancyList descendancy) - { - var children = new HashSet(); + private void AddChildren(PermissionWindow permissionWindow, DescendancyList descendancy) + { + var children = new HashSet(); - foreach (var cacheProgress in AllCacheProgresses) - if (cacheProgress.PermissionWindow_ID == permissionWindow.ID) - children.Add(new PermissionWindowUsedByCacheProgressNode(cacheProgress, permissionWindow, false)); + foreach (var cacheProgress in AllCacheProgresses) + if (cacheProgress.PermissionWindow_ID == permissionWindow.ID) + children.Add(new PermissionWindowUsedByCacheProgressNode(cacheProgress, permissionWindow, false)); - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void AddChildren(AllExternalServersNode allExternalServersNode) - { - AddToDictionaries(new HashSet(AllExternalServers), new DescendancyList(allExternalServersNode)); - } + private void AddChildren(AllExternalServersNode allExternalServersNode) + { + AddToDictionaries(new HashSet(AllExternalServers), new DescendancyList(allExternalServersNode)); + } - private void AddChildren(AllRDMPRemotesNode allRDMPRemotesNode) - { - AddToDictionaries(new HashSet(AllRemoteRDMPs), new DescendancyList(allRDMPRemotesNode)); - } + private void AddChildren(AllRDMPRemotesNode allRDMPRemotesNode) + { + AddToDictionaries(new HashSet(AllRemoteRDMPs), new DescendancyList(allRDMPRemotesNode)); + } - private void AddChildren(AllDashboardsNode allDashboardsNode) - { - AddToDictionaries(new HashSet(AllDashboards), new DescendancyList(allDashboardsNode)); - } + private void AddChildren(AllDashboardsNode allDashboardsNode) + { + AddToDictionaries(new HashSet(AllDashboards), new DescendancyList(allDashboardsNode)); + } - private void AddChildren(AllObjectSharingNode allObjectSharingNode) - { - var descendancy = new DescendancyList(allObjectSharingNode); + private void AddChildren(AllObjectSharingNode allObjectSharingNode) + { + var descendancy = new DescendancyList(allObjectSharingNode); - var allExportsNode = new AllObjectExportsNode(); - var allImportsNode = new AllObjectImportsNode(); + var allExportsNode = new AllObjectExportsNode(); + var allImportsNode = new AllObjectImportsNode(); - AddToDictionaries(new HashSet(AllExports), descendancy.Add(allExportsNode)); - AddToDictionaries(new HashSet(AllImports), descendancy.Add(allImportsNode)); + AddToDictionaries(new HashSet(AllExports), descendancy.Add(allExportsNode)); + AddToDictionaries(new HashSet(AllImports), descendancy.Add(allImportsNode)); - AddToDictionaries(new HashSet(new object[] { allExportsNode, allImportsNode }), descendancy); - } + AddToDictionaries(new HashSet(new object[] { allExportsNode, allImportsNode }), descendancy); + } - /// - /// Creates new s and fills it with all compatible Pipelines - do not call this method more than once - /// - protected void AddPipelineUseCases(Dictionary useCases) - { - var descendancy = new DescendancyList(AllPipelinesNode); - var children = new HashSet(); + /// + /// Creates new s and fills it with all compatible Pipelines - do not call this method more than once + /// + protected void AddPipelineUseCases(Dictionary useCases) + { + var descendancy = new DescendancyList(AllPipelinesNode); + var children = new HashSet(); - //pipelines not found to be part of any StandardPipelineUseCase - var unknownPipelines = new HashSet(AllPipelines); + //pipelines not found to be part of any StandardPipelineUseCase + var unknownPipelines = new HashSet(AllPipelines); - foreach (var useCase in useCases) - { - var node = new StandardPipelineUseCaseNode(useCase.Key, useCase.Value, _commentStore); + foreach (var useCase in useCases) + { + var node = new StandardPipelineUseCaseNode(useCase.Key, useCase.Value, _commentStore); - //keep track of all the use cases - PipelineUseCases.Add(node); + //keep track of all the use cases + PipelineUseCases.Add(node); - foreach (var pipeline in AddChildren(node, descendancy.Add(node))) - unknownPipelines.Remove(pipeline); + foreach (var pipeline in AddChildren(node, descendancy.Add(node))) + unknownPipelines.Remove(pipeline); - children.Add(node); - } + children.Add(node); + } - children.Add(OtherPipelinesNode); - OtherPipelinesNode.Pipelines.AddRange(unknownPipelines.Cast()); - AddToDictionaries(unknownPipelines, descendancy.Add(OtherPipelinesNode)); + children.Add(OtherPipelinesNode); + OtherPipelinesNode.Pipelines.AddRange(unknownPipelines.Cast()); + AddToDictionaries(unknownPipelines, descendancy.Add(OtherPipelinesNode)); - //it is the first standard use case - AddToDictionaries(children, descendancy); - } + //it is the first standard use case + AddToDictionaries(children, descendancy); + } - private IEnumerable AddChildren(StandardPipelineUseCaseNode node, DescendancyList descendancy) - { - var children = new HashSet(); + private IEnumerable AddChildren(StandardPipelineUseCaseNode node, DescendancyList descendancy) + { + var children = new HashSet(); - var repo = new MemoryRepository(); + var repo = new MemoryRepository(); - //Could be an issue here if a pipeline becomes compatible with multiple use cases. - //Should be impossible currently but one day it could be an issue especially if we were to - //support plugin use cases in this hierarchy + //Could be an issue here if a pipeline becomes compatible with multiple use cases. + //Should be impossible currently but one day it could be an issue especially if we were to + //support plugin use cases in this hierarchy - //find compatible pipelines useCase.Value - foreach (var compatiblePipeline in AllPipelines.Where(node.UseCase.GetContext().IsAllowable)) - { - var useCaseNode = new PipelineCompatibleWithUseCaseNode(repo, compatiblePipeline, node.UseCase); + //find compatible pipelines useCase.Value + foreach (var compatiblePipeline in AllPipelines.Where(node.UseCase.GetContext().IsAllowable)) + { + var useCaseNode = new PipelineCompatibleWithUseCaseNode(repo, compatiblePipeline, node.UseCase); - AddChildren(useCaseNode, descendancy.Add(useCaseNode)); + AddChildren(useCaseNode, descendancy.Add(useCaseNode)); - node.Pipelines.Add(compatiblePipeline); - children.Add(useCaseNode); - } + node.Pipelines.Add(compatiblePipeline); + children.Add(useCaseNode); + } - //it is the first standard use case - AddToDictionaries(children, descendancy); + //it is the first standard use case + AddToDictionaries(children, descendancy); - return children.Cast().Select(u => u.Pipeline); - } + return children.Cast().Select(u => u.Pipeline); + } - private void AddChildren(PipelineCompatibleWithUseCaseNode pipelineNode, DescendancyList descendancy) - { - var components = AllPipelineComponents.Where(c => c.Pipeline_ID == pipelineNode.Pipeline.ID) - .OrderBy(o => o.Order) - .ToArray(); + private void AddChildren(PipelineCompatibleWithUseCaseNode pipelineNode, DescendancyList descendancy) + { + var components = AllPipelineComponents.Where(c => c.Pipeline_ID == pipelineNode.Pipeline.ID) + .OrderBy(o => o.Order) + .ToArray(); - foreach (var component in components) - AddChildren(component, descendancy.Add(component)); + foreach (var component in components) + AddChildren(component, descendancy.Add(component)); - var children = new HashSet(components); + var children = new HashSet(components); - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void AddChildren(PipelineComponent pipelineComponent, DescendancyList descendancy) - { - var components = AllPipelineComponentsArguments.Where(c => c.PipelineComponent_ID == pipelineComponent.ID) - .ToArray(); + private void AddChildren(PipelineComponent pipelineComponent, DescendancyList descendancy) + { + var components = AllPipelineComponentsArguments.Where(c => c.PipelineComponent_ID == pipelineComponent.ID) + .ToArray(); - var children = new HashSet(components); + var children = new HashSet(components); - AddToDictionaries(children, descendancy); - } + AddToDictionaries(children, descendancy); + } - private void BuildServerNodes() - { - //add a root node for all the servers to be children of - AllServersNode = new AllServersNode(); + private void BuildServerNodes() + { + //add a root node for all the servers to be children of + AllServersNode = new AllServersNode(); - var descendancy = new DescendancyList(AllServersNode); - var allServers = new List(); + var descendancy = new DescendancyList(AllServersNode); + var allServers = new List(); - foreach (var typeGroup in AllTableInfos.GroupBy(t => t.DatabaseType)) - { - var dbType = typeGroup.Key; - IEnumerable tables = typeGroup; + foreach (var typeGroup in AllTableInfos.GroupBy(t => t.DatabaseType)) + { + var dbType = typeGroup.Key; + IEnumerable tables = typeGroup; - var serversByName = tables - .GroupBy(c => c.Server ?? TableInfoServerNode.NullServerNode, StringComparer.CurrentCultureIgnoreCase) - .Select(s => new TableInfoServerNode(s.Key, dbType, s)); + var serversByName = tables + .GroupBy(c => c.Server ?? TableInfoServerNode.NullServerNode, StringComparer.CurrentCultureIgnoreCase) + .Select(s => new TableInfoServerNode(s.Key, dbType, s)); - foreach (var server in serversByName) - { - allServers.Add(server); - AddChildren(server, descendancy.Add(server)); - } - } + foreach (var server in serversByName) + { + allServers.Add(server); + AddChildren(server, descendancy.Add(server)); + } + } - //create the server nodes - AllServers = allServers.ToArray(); + //create the server nodes + AllServers = allServers.ToArray(); - //record the fact that all the servers are children of the all servers node - AddToDictionaries(new HashSet(AllServers), descendancy); - } + //record the fact that all the servers are children of the all servers node + AddToDictionaries(new HashSet(AllServers), descendancy); + } - private void AddChildren(AllDataAccessCredentialsNode allDataAccessCredentialsNode) - { - var children = new HashSet(); + private void AddChildren(AllDataAccessCredentialsNode allDataAccessCredentialsNode) + { + var children = new HashSet(); - var isKeyMissing = false; - if (_catalogueRepository.EncryptionManager is PasswordEncryptionKeyLocation keyLocation) - isKeyMissing = string.IsNullOrWhiteSpace(keyLocation.GetKeyFileLocation()); + var isKeyMissing = false; + if (_catalogueRepository.EncryptionManager is PasswordEncryptionKeyLocation keyLocation) + isKeyMissing = string.IsNullOrWhiteSpace(keyLocation.GetKeyFileLocation()); - children.Add(new DecryptionPrivateKeyNode(isKeyMissing)); + children.Add(new DecryptionPrivateKeyNode(isKeyMissing)); - foreach (var creds in AllDataAccessCredentials) - children.Add(creds); + foreach (var creds in AllDataAccessCredentials) + children.Add(creds); - AddToDictionaries(children, new DescendancyList(allDataAccessCredentialsNode)); - } + AddToDictionaries(children, new DescendancyList(allDataAccessCredentialsNode)); + } - private void AddChildren(AllANOTablesNode anoTablesNode) - { - AddToDictionaries(new HashSet(AllANOTables), new DescendancyList(anoTablesNode)); - } + private void AddChildren(AllANOTablesNode anoTablesNode) + { + AddToDictionaries(new HashSet(AllANOTables), new DescendancyList(anoTablesNode)); + } - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); - //add catalogues in folder - foreach (var c in folder.ChildObjects) AddChildren(c, descendancy.Add(c)); + //add catalogues in folder + foreach (var c in folder.ChildObjects) AddChildren(c, descendancy.Add(c)); - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); - //add loads in folder - foreach (var lmd in folder.ChildObjects.Where(lmd => lmd.RootLoadMetadata_ID == null).ToArray()) AddChildren(lmd, descendancy.Add(lmd)); - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } + //add loads in folder + foreach (var lmd in folder.ChildObjects.Where(lmd => lmd.RootLoadMetadata_ID == null).ToArray()) AddChildren(lmd, descendancy.Add(lmd)); + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } - private void AddChildren(FolderNode folder, DescendancyList descendancy) + private void AddChildren(FolderNode folder, DescendancyList descendancy) { foreach (var child in folder.ChildFolders) //add subfolder children AddChildren(child, descendancy.Add(child)); - //add loads in folder - foreach (var ds in folder.ChildObjects) AddChildren(ds, descendancy.Add(ds)); + //add loads in folder + foreach (var ds in folder.ChildObjects) AddChildren(ds, descendancy.Add(ds)); - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } - private void AddChildren(FolderNode folder, DescendancyList descendancy) - { - foreach (var child in folder.ChildFolders) - //add subfolder children - AddChildren(child, descendancy.Add(child)); - - - //add cics in folder - foreach (var cic in folder.ChildObjects) AddChildren(cic, descendancy.Add(cic)); + private void AddChildren(FolderNode folder, DescendancyList descendancy) + { + foreach (var child in folder.ChildFolders) + //add subfolder children + AddChildren(child, descendancy.Add(child)); - // Children are the folders + objects - AddToDictionaries(new HashSet( - folder.ChildFolders.Cast() - .Union(folder.ChildObjects)), descendancy - ); - } - - private void AddChildren(Curation.Data.Dataset lmd, DescendancyList descendancy) - { - var childObjects = new List(); - AddToDictionaries(new HashSet(childObjects), descendancy); - } + //add cics in folder + foreach (var cic in folder.ChildObjects) AddChildren(cic, descendancy.Add(cic)); - #region Load Metadata + // Children are the folders + objects + AddToDictionaries(new HashSet( + folder.ChildFolders.Cast() + .Union(folder.ChildObjects)), descendancy + ); + } - private void AddChildren(LoadMetadata lmd, DescendancyList descendancy, bool includeSchedule = true, bool includeCatalogues = true, bool includeVersions = true) + private void AddChildren(Curation.Data.Datasets.Dataset lmd, DescendancyList descendancy) { var childObjects = new List(); - - if (lmd.OverrideRAWServer_ID.HasValue) - { - var server = AllExternalServers.Single(s => s.ID == lmd.OverrideRAWServer_ID.Value); - var usage = new OverrideRawServerNode(lmd, server); - childObjects.Add(usage); - } - if (includeSchedule) - { - var allSchedulesNode = new LoadMetadataScheduleNode(lmd); - AddChildren(allSchedulesNode, descendancy.Add(allSchedulesNode)); - childObjects.Add(allSchedulesNode); - } - - if (includeCatalogues) - { - var allCataloguesNode = new AllCataloguesUsedByLoadMetadataNode(lmd); - AddChildren(allCataloguesNode, descendancy.Add(allCataloguesNode)); - childObjects.Add(allCataloguesNode); - } - - var processTasksNode = new AllProcessTasksUsedByLoadMetadataNode(lmd); - AddChildren(processTasksNode, descendancy.Add(processTasksNode)); - childObjects.Add(processTasksNode); - - if (includeVersions) - { - var versionsNode = new LoadMetadataVersionNode(lmd); - AddChildren(versionsNode, descendancy.Add(versionsNode)); - childObjects.Add(versionsNode); - } - - childObjects.Add(new LoadDirectoryNode(lmd)); - AddToDictionaries(new HashSet(childObjects), descendancy); } - private void AddChildren(LoadMetadataScheduleNode allSchedulesNode, DescendancyList descendancy) - { - var childObjects = new HashSet(); - var lmd = allSchedulesNode.LoadMetadata; + #region Load Metadata + + private void AddChildren(LoadMetadata lmd, DescendancyList descendancy, bool includeSchedule = true, bool includeCatalogues = true, bool includeVersions = true) + { + var childObjects = new List(); + + if (lmd.OverrideRAWServer_ID.HasValue) + { + var server = AllExternalServers.Single(s => s.ID == lmd.OverrideRAWServer_ID.Value); + var usage = new OverrideRawServerNode(lmd, server); + childObjects.Add(usage); + } + if (includeSchedule) + { + var allSchedulesNode = new LoadMetadataScheduleNode(lmd); + AddChildren(allSchedulesNode, descendancy.Add(allSchedulesNode)); + childObjects.Add(allSchedulesNode); + } + + if (includeCatalogues) + { + var allCataloguesNode = new AllCataloguesUsedByLoadMetadataNode(lmd); + AddChildren(allCataloguesNode, descendancy.Add(allCataloguesNode)); + childObjects.Add(allCataloguesNode); + } + + var processTasksNode = new AllProcessTasksUsedByLoadMetadataNode(lmd); + AddChildren(processTasksNode, descendancy.Add(processTasksNode)); + childObjects.Add(processTasksNode); + + if (includeVersions) + { + var versionsNode = new LoadMetadataVersionNode(lmd); + AddChildren(versionsNode, descendancy.Add(versionsNode)); + childObjects.Add(versionsNode); + } + + childObjects.Add(new LoadDirectoryNode(lmd)); + + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + private void AddChildren(LoadMetadataScheduleNode allSchedulesNode, DescendancyList descendancy) + { + var childObjects = new HashSet(); + + var lmd = allSchedulesNode.LoadMetadata; + + foreach (var lp in AllLoadProgresses.Where(p => p.LoadMetadata_ID == lmd.ID)) + { + AddChildren(lp, descendancy.Add(lp)); + childObjects.Add(lp); + } + + if (childObjects.Any()) + AddToDictionaries(childObjects, descendancy); + } + + private void AddChildren(LoadProgress loadProgress, DescendancyList descendancy) + { + var cacheProgresses = AllCacheProgresses.Where(cp => cp.LoadProgress_ID == loadProgress.ID).ToArray(); + + foreach (var cacheProgress in cacheProgresses) + AddChildren(cacheProgress, descendancy.Add(cacheProgress)); + + if (cacheProgresses.Any()) + AddToDictionaries(new HashSet(cacheProgresses), descendancy); + } + + private void AddChildren(CacheProgress cacheProgress, DescendancyList descendancy) + { + var children = new HashSet(); + + if (cacheProgress.PermissionWindow_ID != null) + { + var window = AllPermissionWindows.Single(w => w.ID == cacheProgress.PermissionWindow_ID); + var windowNode = new PermissionWindowUsedByCacheProgressNode(cacheProgress, window, true); + + children.Add(windowNode); + } + + if (children.Any()) + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AllProcessTasksUsedByLoadMetadataNode allProcessTasksUsedByLoadMetadataNode, + DescendancyList descendancy) + { + var childObjects = new HashSet(); + + var lmd = allProcessTasksUsedByLoadMetadataNode.LoadMetadata; + childObjects.Add(new LoadStageNode(lmd, LoadStage.GetFiles)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.Mounting)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustRaw)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustStaging)); + childObjects.Add(new LoadStageNode(lmd, LoadStage.PostLoad)); + + foreach (LoadStageNode node in childObjects) + AddChildren(node, descendancy.Add(node)); + + AddToDictionaries(childObjects, descendancy); + } + + private void AddChildren(LoadStageNode loadStageNode, DescendancyList descendancy) + { + var tasks = AllProcessTasks.Where( + p => p.LoadMetadata_ID == loadStageNode.LoadMetadata.ID && p.LoadStage == loadStageNode.LoadStage) + .OrderBy(o => o.Order).ToArray(); + + foreach (var processTask in tasks) + AddChildren(processTask, descendancy.Add(processTask)); + + if (tasks.Any()) + AddToDictionaries(new HashSet(tasks), descendancy); + } + + private void AddChildren(ProcessTask procesTask, DescendancyList descendancy) + { + var args = AllProcessTasksArguments.Where( + a => a.ProcessTask_ID == procesTask.ID).ToArray(); + + if (args.Any()) + AddToDictionaries(new HashSet(args), descendancy); + } + + private void AddChildren(LoadMetadataVersionNode LoadMetadataVersionNode, DescendancyList descendancy) + { + LoadMetadataVersionNode.LoadMetadataVersions = AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID == LoadMetadataVersionNode.LoadMetadata.ID).ToList(); + var childObjects = new List(); + + foreach (var lmd in LoadMetadataVersionNode.LoadMetadataVersions) + { + AddChildren(lmd, descendancy.Add(lmd), false, false, false); + childObjects.Add(lmd); + } + AddToDictionaries(new HashSet(childObjects), descendancy); + + } + + private void AddChildren(AllCataloguesUsedByLoadMetadataNode allCataloguesUsedByLoadMetadataNode, + DescendancyList descendancy) + { + var loadMetadataId = allCataloguesUsedByLoadMetadataNode.LoadMetadata.ID; + var linkedCatalogueIDs = AllLoadMetadataLinkage.Where(link => link.LoadMetadataID == loadMetadataId).Select(static link => link.CatalogueID); + var usedCatalogues = linkedCatalogueIDs.Select(catalogueId => AllCatalogues.FirstOrDefault(c => c.ID == catalogueId)).Where(static foundCatalogue => foundCatalogue is not null).ToList(); + allCataloguesUsedByLoadMetadataNode.UsedCatalogues = usedCatalogues; + var childObjects = usedCatalogues.Select(foundCatalogue => new CatalogueUsedByLoadMetadataNode(allCataloguesUsedByLoadMetadataNode.LoadMetadata, foundCatalogue)).Cast().ToHashSet(); + + AddToDictionaries(childObjects, descendancy); + } + + #endregion + + protected void AddChildren(Catalogue c, DescendancyList descendancy) + { + var childObjects = new List(); + + var catalogueAggregates = AllAggregateConfigurations.Where(a => a.Catalogue_ID == c.ID).ToArray(); + var cohortAggregates = catalogueAggregates.Where(a => a.IsCohortIdentificationAggregate).ToArray(); + var regularAggregates = catalogueAggregates.Except(cohortAggregates).ToArray(); + + //get all the CatalogueItems for this Catalogue (TryGet because Catalogue may not have any items + var cis = _catalogueToCatalogueItems.TryGetValue(c.ID, out var result) + ? result.ToArray() + : Array.Empty(); + + //tell the CatalogueItems that we are are their parent + foreach (var ci in cis) + ci.InjectKnown(c); + + // core includes project specific which basically means the same thing + var core = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Core || + ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.ProjectSpecific) + , ExtractionCategory.Core); + + c.InjectKnown(cis); + + var deprecated = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Deprecated), + ExtractionCategory.Deprecated); + var special = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.SpecialApprovalRequired), + ExtractionCategory.SpecialApprovalRequired); + var intern = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Internal), + ExtractionCategory.Internal); + var supplemental = new CatalogueItemsNode(c, + cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Supplemental), + ExtractionCategory.Supplemental); + var notExtractable = new CatalogueItemsNode(c, cis.Where(ci => ci.ExtractionInformation == null), null); + + AddChildren(core, descendancy.Add(core)); + childObjects.Add(core); + + foreach (var optional in new[] { deprecated, special, intern, supplemental, notExtractable }) + if (optional.CatalogueItems.Any()) + { + AddChildren(optional, descendancy.Add(optional)); + childObjects.Add(optional); + } + + //do we have any foreign key fields into this lookup table + var lookups = AllLookups.Where(l => c.CatalogueItems.Any(ci => ci.ColumnInfo_ID == l.ForeignKey_ID)).ToArray(); + + var docs = AllSupportingDocuments.Where(d => d.Catalogue_ID == c.ID).ToArray(); + var sql = AllSupportingSQL.Where(d => d.Catalogue_ID == c.ID).ToArray(); + + //if there are supporting documents or supporting sql files then add documentation node + if (docs.Any() || sql.Any()) + { + var documentationNode = new DocumentationNode(c, docs, sql); + + //add the documentations node + childObjects.Add(documentationNode); + + //record the children + AddToDictionaries(new HashSet(docs.Cast().Union(sql)), descendancy.Add(documentationNode)); + } + + if (lookups.Any()) + { + var lookupsNode = new CatalogueLookupsNode(c, lookups); + //add the documentations node + childObjects.Add(lookupsNode); + + + //record the children + AddToDictionaries(new HashSet(lookups.Select(l => new CatalogueLookupUsageNode(c, l))), + descendancy.Add(lookupsNode)); + } + + if (regularAggregates.Any()) + { + var aggregatesNode = new AggregatesNode(c, regularAggregates); + childObjects.Add(aggregatesNode); + + var nodeDescendancy = descendancy.Add(aggregatesNode); + AddToDictionaries(new HashSet(regularAggregates), nodeDescendancy); + + foreach (var regularAggregate in regularAggregates) + AddChildren(regularAggregate, nodeDescendancy.Add(regularAggregate)); + } + + //finalise + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + private void InjectCatalogueItems() + { + foreach (var ci in AllCatalogueItems) + if (_extractionInformationsByCatalogueItem.TryGetValue(ci.ID, out var ei)) + ci.InjectKnown(ei); + else + ci.InjectKnown((ExtractionInformation)null); + } + + private void AddChildren(CatalogueItemsNode node, DescendancyList descendancyList) + { + AddToDictionaries(new HashSet(node.CatalogueItems), descendancyList); + + foreach (var ci in node.CatalogueItems) + AddChildren(ci, descendancyList.Add(ci)); + } + + private void AddChildren(AggregateConfiguration aggregateConfiguration, DescendancyList descendancy) + { + var childrenObjects = new HashSet(); + + var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(aggregateConfiguration)).Cast() + .ToArray(); + + foreach (var p in parameters) + childrenObjects.Add(p); + + // show the dimensions in the tree + foreach (var dim in aggregateConfiguration.AggregateDimensions) childrenObjects.Add(dim); + + // show the axis (if any) in the tree. If there are multiple axis in this tree then that is bad but maybe the user can delete one of them to fix the situation + foreach (var axis in AllAggregateContinuousDateAxis.Where(a => + aggregateConfiguration.AggregateDimensions.Any(d => d.ID == a.AggregateDimension_ID))) + childrenObjects.Add(axis); + + //we can step into this twice, once via Catalogue children and once via CohortIdentificationConfiguration children + //if we get in via Catalogue children then descendancy will be Ignore=true we don't end up emphasising into CatalogueCollectionUI when + //really user wants to see it in CohortIdentificationCollectionUI + if (aggregateConfiguration.RootFilterContainer_ID != null) + { + var container = AllAggregateContainersDictionary[(int)aggregateConfiguration.RootFilterContainer_ID]; + + AddChildren(container, descendancy.Add(container)); + childrenObjects.Add(container); + } + + AddToDictionaries(childrenObjects, descendancy); + } + + private void AddChildren(AggregateFilterContainer container, DescendancyList descendancy) + { + var childrenObjects = new List(); + + var subcontainers = _aggregateFilterManager.GetSubContainers(container); + var filters = _aggregateFilterManager.GetFilters(container); + + foreach (AggregateFilterContainer subcontainer in subcontainers) + { + //one of our children is this subcontainer + childrenObjects.Add(subcontainer); + + //but also document its children + AddChildren(subcontainer, descendancy.Add(subcontainer)); + } + + //also add the filters for the container + foreach (var f in filters) + { + // for filters add the parameters under them + AddChildren((AggregateFilter)f, descendancy.Add(f)); + childrenObjects.Add(f); + } + + //add our children to the dictionary + AddToDictionaries(new HashSet(childrenObjects), descendancy); + } + + private void AddChildren(AggregateFilter f, DescendancyList descendancy) + { + AddToDictionaries(new HashSet(AllAggregateFilterParameters.Where(p => p.AggregateFilter_ID == f.ID)), + descendancy); + } + + private void AddChildren(CatalogueItem ci, DescendancyList descendancy) + { + var childObjects = new List(); + + var ei = ci.ExtractionInformation; + if (ei != null) + { + childObjects.Add(ei); + AddChildren(ei, descendancy.Add(ei)); + } + else + { + ci.InjectKnown( + (ExtractionInformation)null); // we know the CatalogueItem has no ExtractionInformation child because it's not in the dictionary + } + + if (ci.ColumnInfo_ID.HasValue && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) + childObjects.Add(new LinkedColumnInfoNode(ci, col)); + + AddToDictionaries(new HashSet(childObjects), descendancy); + } + + private void AddChildren(ExtractionInformation extractionInformation, DescendancyList descendancy) + { + var children = new HashSet(); + + foreach (var filter in AllCatalogueFilters.Where(f => f.ExtractionInformation_ID == extractionInformation.ID)) + { + //add the filter as a child of the + children.Add(filter); + AddChildren(filter, descendancy.Add(filter)); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(ExtractionFilter filter, DescendancyList descendancy) + { + var children = new HashSet(); + var parameters = AllCatalogueParameters.Where(p => p.ExtractionFilter_ID == filter.ID).ToArray(); + var parameterSets = AllCatalogueValueSets.Where(vs => vs.ExtractionFilter_ID == filter.ID).ToArray(); + + filter.InjectKnown(parameterSets); + + foreach (var p in parameters) + children.Add(p); + + foreach (var set in parameterSets) + { + children.Add(set); + AddChildren(set, descendancy.Add(set), parameters); + } + + if (children.Any()) + AddToDictionaries(children, descendancy); + } + + private void AddChildren(ExtractionFilterParameterSet set, DescendancyList descendancy, + ExtractionFilterParameter[] filterParameters) + { + var children = new HashSet(); + + foreach (var setValue in AllCatalogueValueSetValues.Where(v => v.ExtractionFilterParameterSet_ID == set.ID)) + { + setValue.InjectKnown(filterParameters.SingleOrDefault(p => p.ID == setValue.ExtractionFilterParameter_ID)); + children.Add(setValue); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(CohortIdentificationConfiguration cic, DescendancyList descendancy) + { + var children = new HashSet(); + + //it has an associated query cache + if (cic.QueryCachingServer_ID != null) + children.Add(new QueryCacheUsedByCohortIdentificationNode(cic, + AllExternalServers.Single(s => s.ID == cic.QueryCachingServer_ID))); + + var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(cic)).Cast().ToArray(); + foreach (var p in parameters) children.Add(p); + + //if it has a root container + if (cic.RootCohortAggregateContainer_ID != null) + { + var container = AllCohortAggregateContainers.Single(c => c.ID == cic.RootCohortAggregateContainer_ID); + AddChildren(container, descendancy.Add(container).SetBetterRouteExists()); + children.Add(container); + } + + //get the patient index tables + var joinableNode = new JoinableCollectionNode(cic, + AllJoinables.Where(j => j.CohortIdentificationConfiguration_ID == cic.ID).ToArray()); + AddChildren(joinableNode, descendancy.Add(joinableNode).SetBetterRouteExists()); + children.Add(joinableNode); + + AddToDictionaries(children, descendancy.SetBetterRouteExists()); + } + + private void AddChildren(JoinableCollectionNode joinablesNode, DescendancyList descendancy) + { + var children = new HashSet(); + + foreach (var joinable in joinablesNode.Joinables) + try + { + var agg = AllAggregateConfigurations.Single(ac => ac.ID == joinable.AggregateConfiguration_ID); + ForceAggregateNaming(agg, descendancy); + children.Add(agg); + + //it's no longer an orphan because it's in a known cic (as a patient index table) + OrphanAggregateConfigurations.Remove(agg); + + AddChildren(agg, descendancy.Add(agg)); + } + catch (Exception e) + { + throw new Exception( + $"JoinableCohortAggregateConfiguration (patient index table) object (ID={joinable.ID}) references AggregateConfiguration_ID {joinable.AggregateConfiguration_ID} but that AggregateConfiguration was not found", + e); + } + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(CohortAggregateContainer container, DescendancyList descendancy) + { + //get subcontainers + var subcontainers = _cohortContainerManager.GetChildren(container).OfType().ToList(); + + //if there are subcontainers + foreach (var subcontainer in subcontainers) + AddChildren(subcontainer, descendancy.Add(subcontainer)); + + //get our configurations + var configurations = _cohortContainerManager.GetChildren(container).OfType().ToList(); + + //record the configurations children including full descendancy + foreach (var configuration in configurations) + { + ForceAggregateNaming(configuration, descendancy); + AddChildren(configuration, descendancy.Add(configuration)); + + //it's no longer an orphan because it's in a known cic + OrphanAggregateConfigurations.Remove(configuration); + } + + //all our children (containers and aggregates) + //children are all aggregates and containers at the current hierarchy level in order + var children = subcontainers.Union(configurations.Cast()).OrderBy(o => o.Order).ToList(); + + AddToDictionaries(new HashSet(children), descendancy); + } + + private void ForceAggregateNaming(AggregateConfiguration configuration, DescendancyList descendancy) + { + //configuration has the wrong name + if (!configuration.IsCohortIdentificationAggregate) + { + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( + $"Had to fix naming of configuration '{configuration}' because it didn't start with correct cic prefix", + CheckResult.Warning)); + descendancy.Parents.OfType().Single() + .EnsureNamingConvention(configuration); + configuration.SaveToDatabase(); + } + } + + private void AddChildren(TableInfoServerNode serverNode, DescendancyList descendancy) + { + //add empty hashset + var children = new HashSet(); + + var databases = + serverNode.Tables.GroupBy( + k => k.Database ?? TableInfoDatabaseNode.NullDatabaseNode, StringComparer.CurrentCultureIgnoreCase) + .Select(g => new TableInfoDatabaseNode(g.Key, serverNode, g)); + + foreach (var db in databases) + { + children.Add(db); + AddChildren(db, descendancy.Add(db)); + } + + //now we have recorded all the children add them with descendancy + AddToDictionaries(children, descendancy); + } + + private void AddChildren(TableInfoDatabaseNode dbNode, DescendancyList descendancy) + { + //add empty hashset + var children = new HashSet(); + + foreach (var t in dbNode.Tables) + { + //record the children of the table infos (mostly column infos) + children.Add(t); + + //the all servers node=>the TableInfoServerNode => the t + AddChildren(t, descendancy.Add(t)); + } + + //now we have recorded all the children add them with descendancy + AddToDictionaries(children, descendancy); + } + + private void AddChildren(TableInfo tableInfo, DescendancyList descendancy) + { + //add empty hashset + var children = new HashSet(); + + //if the table has an identifier dump listed + if (tableInfo.IdentifierDumpServer_ID != null) + { + //if there is a dump (e.g. for dilution and dumping - not appearing in the live table) + var server = AllExternalServers.Single(s => s.ID == tableInfo.IdentifierDumpServer_ID.Value); + + children.Add(new IdentifierDumpServerUsageNode(tableInfo, server)); + } + + //get the discarded columns in this table + var discardedCols = new HashSet(AllPreLoadDiscardedColumns.Where(c => c.TableInfo_ID == tableInfo.ID)); + + //tell the column who their parent is so they don't need to look up the database + foreach (PreLoadDiscardedColumn discardedCol in discardedCols) + discardedCol.InjectKnown(tableInfo); + + //if there are discarded columns + if (discardedCols.Any()) + { + var identifierDumpNode = new PreLoadDiscardedColumnsNode(tableInfo); + + //record that the usage is a child of TableInfo + children.Add(identifierDumpNode); + + //record that the discarded columns are children of identifier dump usage node + AddToDictionaries(discardedCols, descendancy.Add(identifierDumpNode)); + } + + //if it is a table valued function + if (tableInfo.IsTableValuedFunction) + { + //that has parameters + var parameters = tableInfo.GetAllParameters(); + + foreach (var p in parameters) children.Add(p); + } + + //next add the column infos + if (TableInfosToColumnInfos.TryGetValue(tableInfo.ID, out var result)) + foreach (var c in result) + { + children.Add(c); + c.InjectKnown(tableInfo); + AddChildren(c, descendancy.Add(c).SetBetterRouteExists()); + } + + //finally add any credentials objects + if (AllDataAccessCredentialUsages.TryGetValue(tableInfo, out var nodes)) + foreach (var node in nodes) + children.Add(node); + + //now we have recorded all the children add them with descendancy via the TableInfo descendancy + AddToDictionaries(children, descendancy); + } + + private void AddChildren(ColumnInfo columnInfo, DescendancyList descendancy) + { + var lookups = AllLookups.Where(l => l.Description_ID == columnInfo.ID).ToArray(); + var joinInfos = AllJoinInfos.Where(j => j.PrimaryKey_ID == columnInfo.ID); + + var children = new HashSet(); + + foreach (var l in lookups) + children.Add(l); + + foreach (var j in joinInfos) + children.Add(j); + + if (children.Any()) + AddToDictionaries(children, descendancy); + } + + + protected void AddToDictionaries(HashSet children, DescendancyList list) + { + if (list.IsEmpty) + throw new ArgumentException("DescendancyList cannot be empty", nameof(list)); + + //document that the last parent has these as children + var parent = list.Last(); + + _childDictionary.AddOrUpdate(parent, + children, (p, s) => children); + + //now document the entire parent order to reach each child object i.e. 'Root=>Grandparent=>Parent' is how you get to 'Child' + foreach (var o in children) + _descendancyDictionary.AddOrUpdate(o, list, (k, v) => HandleDescendancyCollision(k, v, list)); + + + foreach (var masquerader in children.OfType()) + { + var key = masquerader.MasqueradingAs(); + + if (!AllMasqueraders.ContainsKey(key)) + AllMasqueraders.AddOrUpdate(key, new HashSet(), (o, set) => set); + + lock (AllMasqueraders) + { + AllMasqueraders[key].Add(masquerader); + } + } + } + + private static DescendancyList HandleDescendancyCollision(object key, DescendancyList oldRoute, + DescendancyList newRoute) + { + //if the new route is the best best + if (newRoute.NewBestRoute && !oldRoute.NewBestRoute) + return newRoute; + + // If the new one is marked BetterRouteExists just throw away the new one + return newRoute.BetterRouteExists ? oldRoute : newRoute; + // If in doubt use the newest one + } + + private HashSet GetAllObjects() + { + //anything which has children or is a child of someone else (distinct because HashSet) + return new HashSet(_childDictionary.SelectMany(kvp => kvp.Value).Union(_childDictionary.Keys)); + } + + public virtual object[] GetChildren(object model) + { + lock (WriteLock) + { + ; //if we have a record of any children in the child dictionary for the parent model object + if (_childDictionary.TryGetValue(model, out var cached)) + return cached.OrderBy(static o => o.ToString()).ToArray(); + + return model switch + { + //if they want the children of a Pipeline (which we don't track) just serve the components + Pipeline p => p.PipelineComponents.ToArray(), + //if they want the children of a PipelineComponent (which we don't track) just serve the arguments + PipelineComponent pc => pc.PipelineComponentArguments.ToArray(), + _ => Array.Empty() + }; + } + } + + public IEnumerable GetAllObjects(Type type, bool unwrapMasqueraders) + { + lock (WriteLock) + { + //things that are a match on Type but not IMasqueradeAs + var exactMatches = GetAllSearchables().Keys.Where(t => t is not IMasqueradeAs).Where(type.IsInstanceOfType); + + //Union the unwrapped masqueraders + return unwrapMasqueraders + ? exactMatches.Union( + AllMasqueraders + .Select(kvp => kvp.Key) + .OfType() + .Where(type.IsInstanceOfType)) + .Distinct() + : exactMatches; + } + } + + public DescendancyList GetDescendancyListIfAnyFor(object model) + { + lock (WriteLock) + { + return _descendancyDictionary.GetValueOrDefault(model); + } + } + + + public object GetRootObjectOrSelf(object objectToEmphasise) + { + lock (WriteLock) + { + var descendancy = GetDescendancyListIfAnyFor(objectToEmphasise); + + return descendancy != null && descendancy.Parents.Any() ? descendancy.Parents[0] : objectToEmphasise; + } + } + + + public virtual Dictionary GetAllSearchables() + { + lock (WriteLock) + { + var toReturn = new Dictionary(); + + foreach (var kvp in _descendancyDictionary.Where(kvp => kvp.Key is IMapsDirectlyToDatabaseTable)) + toReturn.Add((IMapsDirectlyToDatabaseTable)kvp.Key, kvp.Value); + + return toReturn; + } + } + + public IEnumerable GetAllChildrenRecursively(object o) + { + lock (WriteLock) + { + var toReturn = new List(); + + foreach (var child in GetChildren(o)) + { + toReturn.Add(child); + toReturn.AddRange(GetAllChildrenRecursively(child)); + } + + return toReturn; + } + } + + /// + /// Asks all plugins to provide the child objects for every object we have found so far. This method is recursive, call it with null the first time to use all objects. It will then + /// call itself with all the new objects that were sent back by the plugin (so that new objects found can still have children). + /// + /// + protected void GetPluginChildren(HashSet objectsToAskAbout = null) + { + lock (WriteLock) + { + var newObjectsFound = new HashSet(); + + var sw = new Stopwatch(); + + var providers = _pluginChildProviders.Except(_blockedPlugins).ToArray(); + + //for every object found so far + if (providers.Any()) + foreach (var o in objectsToAskAbout ?? GetAllObjects()) + //for every plugin loaded (that is not forbidlisted) + foreach (var plugin in providers) + //ask about the children + try + { + sw.Restart(); + //otherwise ask plugin what its children are + var pluginChildren = plugin.GetChildren(o); + + //if the plugin takes too long to respond we need to stop + if (sw.ElapsedMilliseconds > 1000) + { + _blockedPlugins.Add(plugin); + throw new Exception( + $"Plugin '{plugin}' was forbidlisted for taking too long to respond to GetChildren(o) where o was a '{o.GetType().Name}' ('{o}')"); + } + + //it has children + if (pluginChildren != null && pluginChildren.Any()) + { + //get the descendancy of the parent + var parentDescendancy = GetDescendancyListIfAnyFor(o); + var newDescendancy = parentDescendancy == null + ? new DescendancyList(new[] { o }) + : //if the parent is a root level object start a new descendancy list from it + parentDescendancy + .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one + newDescendancy = + parentDescendancy + .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one + + //record that + foreach (var pluginChild in pluginChildren) + { + //if the parent didn't have any children before + if (!_childDictionary.ContainsKey(o)) + _childDictionary.AddOrUpdate(o, new HashSet(), + (o1, set) => set); //it does now + + + //add us to the parent objects child collection + _childDictionary[o].Add(pluginChild); + + //add to the child collection of the parent object kvp.Key + _descendancyDictionary.AddOrUpdate(pluginChild, newDescendancy, + (s, e) => newDescendancy); + + //we have found a new object so we must ask other plugins about it (chances are a plugin will have a whole tree of sub objects) + newObjectsFound.Add(pluginChild); + } + } + } + catch (Exception e) + { + _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail, e)); + } + + if (newObjectsFound.Any()) + GetPluginChildren(newObjectsFound); + } + } + + public IEnumerable GetMasqueradersOf(object o) + { + lock (WriteLock) + { + return AllMasqueraders.TryGetValue(o, out var result) ? result : Array.Empty(); + } + } + + protected T[] GetAllObjects(IRepository repository) where T : IMapsDirectlyToDatabaseTable + { + lock (WriteLock) + { + return repository.GetAllObjects(); + } + } + + + protected void AddToReturnSearchablesWithNoDecendancy( + Dictionary toReturn, + IEnumerable toAdd) + { + lock (WriteLock) + { + foreach (var m in toAdd) + toReturn.Add(m, null); + } + } + + public virtual void UpdateTo(ICoreChildProvider other) + { + ArgumentNullException.ThrowIfNull(other); + + if (other is not CatalogueChildProvider otherCat) + throw new NotSupportedException( + $"Did not know how to UpdateTo ICoreChildProvider of type {other.GetType().Name}"); + + AllLoadMetadatas = otherCat.AllLoadMetadatas; + AllProcessTasks = otherCat.AllProcessTasks; + AllProcessTasksArguments = otherCat.AllProcessTasksArguments; + AllLoadProgresses = otherCat.AllLoadProgresses; + AllCacheProgresses = otherCat.AllCacheProgresses; + AllPermissionWindows = otherCat.AllPermissionWindows; + AllCatalogues = otherCat.AllCatalogues; + AllCataloguesDictionary = otherCat.AllCataloguesDictionary; + AllSupportingDocuments = otherCat.AllSupportingDocuments; + AllSupportingSQL = otherCat.AllSupportingSQL; + _childDictionary = otherCat._childDictionary; + _descendancyDictionary = otherCat._descendancyDictionary; + _catalogueToCatalogueItems = otherCat._catalogueToCatalogueItems; + AllCatalogueItemsDictionary = otherCat.AllCatalogueItemsDictionary; + _allColumnInfos = otherCat._allColumnInfos; + AllAggregateConfigurations = otherCat.AllAggregateConfigurations; + AllAggregateDimensions = otherCat.AllAggregateDimensions; + AllAggregateContinuousDateAxis = otherCat.AllAggregateContinuousDateAxis; + AllRDMPRemotesNode = otherCat.AllRDMPRemotesNode; + AllRemoteRDMPs = otherCat.AllRemoteRDMPs; + AllDashboardsNode = otherCat.AllDashboardsNode; + AllDashboards = otherCat.AllDashboards; + AllObjectSharingNode = otherCat.AllObjectSharingNode; + AllImports = otherCat.AllImports; + AllExports = otherCat.AllExports; + AllStandardRegexesNode = otherCat.AllStandardRegexesNode; + AllPipelinesNode = otherCat.AllPipelinesNode; + OtherPipelinesNode = otherCat.OtherPipelinesNode; + AllPipelines = otherCat.AllPipelines; + AllPipelineComponents = otherCat.AllPipelineComponents; + AllPipelineComponentsArguments = otherCat.AllPipelineComponentsArguments; + AllStandardRegexes = otherCat.AllStandardRegexes; + AllANOTablesNode = otherCat.AllANOTablesNode; + AllANOTables = otherCat.AllANOTables; + AllExternalServers = otherCat.AllExternalServers; + AllServers = otherCat.AllServers; + AllTableInfos = otherCat.AllTableInfos; + AllDataAccessCredentialsNode = otherCat.AllDataAccessCredentialsNode; + AllExternalServersNode = otherCat.AllExternalServersNode; + AllServersNode = otherCat.AllServersNode; + AllDataAccessCredentials = otherCat.AllDataAccessCredentials; + AllDataAccessCredentialUsages = otherCat.AllDataAccessCredentialUsages; + TableInfosToColumnInfos = otherCat.TableInfosToColumnInfos; + AllColumnInfos = otherCat.AllColumnInfos; + AllPreLoadDiscardedColumns = otherCat.AllPreLoadDiscardedColumns; + AllLookups = otherCat.AllLookups; + AllJoinInfos = otherCat.AllJoinInfos; + AllAnyTableParameters = otherCat.AllAnyTableParameters; + AllMasqueraders = otherCat.AllMasqueraders; + AllExtractionInformationsDictionary = otherCat.AllExtractionInformationsDictionary; + _pluginChildProviders = otherCat._pluginChildProviders; + AllPermissionWindowsNode = otherCat.AllPermissionWindowsNode; + LoadMetadataRootFolder = otherCat.LoadMetadataRootFolder; + CatalogueRootFolder = otherCat.CatalogueRootFolder; + CohortIdentificationConfigurationRootFolder = otherCat.CohortIdentificationConfigurationRootFolder; + AllConnectionStringKeywordsNode = otherCat.AllConnectionStringKeywordsNode; + AllConnectionStringKeywords = otherCat.AllConnectionStringKeywords; + AllAggregateContainersDictionary = otherCat.AllAggregateContainersDictionary; + AllAggregateFilters = otherCat.AllAggregateFilters; + AllAggregateFilterParameters = otherCat.AllAggregateFilterParameters; + AllCohortIdentificationConfigurations = otherCat.AllCohortIdentificationConfigurations; + AllCohortAggregateContainers = otherCat.AllCohortAggregateContainers; + AllJoinables = otherCat.AllJoinables; + AllJoinUses = otherCat.AllJoinUses; + AllGovernanceNode = otherCat.AllGovernanceNode; + AllGovernancePeriods = otherCat.AllGovernancePeriods; + AllGovernanceDocuments = otherCat.AllGovernanceDocuments; + GovernanceCoverage = otherCat.GovernanceCoverage; + AllJoinableCohortAggregateConfigurationUse = otherCat.AllJoinableCohortAggregateConfigurationUse; + AllPluginsNode = otherCat.AllPluginsNode; + PipelineUseCases = otherCat.PipelineUseCases; + OrphanAggregateConfigurationsNode = otherCat.OrphanAggregateConfigurationsNode; + TemplateAggregateConfigurationsNode = otherCat.TemplateAggregateConfigurationsNode; + AllCatalogueParameters = otherCat.AllCatalogueParameters; + AllCatalogueValueSets = otherCat.AllCatalogueValueSets; + AllCatalogueValueSetValues = otherCat.AllCatalogueValueSetValues; + OrphanAggregateConfigurations = otherCat.OrphanAggregateConfigurations; + AllTemplateCohortIdentificationConfigurationsNode = other.AllTemplateCohortIdentificationConfigurationsNode; + + } + + public virtual bool SelectiveRefresh(IMapsDirectlyToDatabaseTable databaseEntity) + { + ProgressStopwatch.Restart(); + + return databaseEntity switch + { + AggregateFilterParameter afp => SelectiveRefresh(afp.AggregateFilter), + AggregateFilter af => SelectiveRefresh(af), + AggregateFilterContainer afc => SelectiveRefresh(afc), + CohortAggregateContainer cac => SelectiveRefresh(cac), + ExtractionInformation ei => SelectiveRefresh(ei), + CatalogueItem ci => SelectiveRefresh(ci), + _ => false + }; + } + + + public bool SelectiveRefresh(CatalogueItem ci) + { + var descendancy = GetDescendancyListIfAnyFor(ci.Catalogue); + if (descendancy == null) return false; + + FetchCatalogueItems(); + FetchExtractionInformations(); + AddChildren(ci.Catalogue, descendancy.Add(ci.Catalogue)); + return true; + } + + public bool SelectiveRefresh(ExtractionInformation ei) + { + var descendancy = GetDescendancyListIfAnyFor(ei); + var cata = descendancy?.Parents.OfType().LastOrDefault() ?? ei.CatalogueItem.Catalogue; + + if (cata == null) + return false; + + var cataDescendancy = GetDescendancyListIfAnyFor(cata); + + if (cataDescendancy == null) + return false; + + FetchCatalogueItems(); + + foreach (var ci in AllCatalogueItems.Where(ci => ci.ID == ei.CatalogueItem_ID)) ci.ClearAllInjections(); + + // property changes or deleting the ExtractionInformation + FetchExtractionInformations(); + + // refresh the Catalogue + AddChildren(cata, cataDescendancy.Add(cata)); + return true; + } + + public bool SelectiveRefresh(CohortAggregateContainer container) + { + var parentContainer = container.GetParentContainerIfAny(); + if (parentContainer != null) + { + var descendancy = GetDescendancyListIfAnyFor(parentContainer); + + if (descendancy != null) + { + BuildAggregateConfigurations(); + + BuildCohortCohortAggregateContainers(); + AddChildren(parentContainer, descendancy.Add(parentContainer)); + return true; + } + } + + var cic = container.GetCohortIdentificationConfiguration(); + + if (cic != null) + { + var descendancy = GetDescendancyListIfAnyFor(cic); + + if (descendancy != null) + { + BuildAggregateConfigurations(); + BuildCohortCohortAggregateContainers(); + AddChildren(cic, descendancy.Add(cic)); + return true; + } + } + + return false; + } + + public bool SelectiveRefresh(AggregateFilter f) + { + var knownContainer = GetDescendancyListIfAnyFor(f.FilterContainer); + if (knownContainer == null) return false; + + BuildAggregateFilterContainers(); + AddChildren((AggregateFilterContainer)f.FilterContainer, knownContainer.Add(f.FilterContainer)); + return true; + } - foreach (var lp in AllLoadProgresses.Where(p => p.LoadMetadata_ID == lmd.ID)) - { - AddChildren(lp, descendancy.Add(lp)); - childObjects.Add(lp); - } + public bool SelectiveRefresh(AggregateFilterContainer container) + { + var aggregate = container.GetAggregate(); - if (childObjects.Any()) - AddToDictionaries(childObjects, descendancy); - } + if (aggregate == null) return false; - private void AddChildren(LoadProgress loadProgress, DescendancyList descendancy) - { - var cacheProgresses = AllCacheProgresses.Where(cp => cp.LoadProgress_ID == loadProgress.ID).ToArray(); - - foreach (var cacheProgress in cacheProgresses) - AddChildren(cacheProgress, descendancy.Add(cacheProgress)); - - if (cacheProgresses.Any()) - AddToDictionaries(new HashSet(cacheProgresses), descendancy); - } - - private void AddChildren(CacheProgress cacheProgress, DescendancyList descendancy) - { - var children = new HashSet(); + var descendancy = GetDescendancyListIfAnyFor(aggregate); - if (cacheProgress.PermissionWindow_ID != null) - { - var window = AllPermissionWindows.Single(w => w.ID == cacheProgress.PermissionWindow_ID); - var windowNode = new PermissionWindowUsedByCacheProgressNode(cacheProgress, window, true); + if (descendancy == null) return false; - children.Add(windowNode); - } - - if (children.Any()) - AddToDictionaries(children, descendancy); - } - - private void AddChildren(AllProcessTasksUsedByLoadMetadataNode allProcessTasksUsedByLoadMetadataNode, - DescendancyList descendancy) - { - var childObjects = new HashSet(); - - var lmd = allProcessTasksUsedByLoadMetadataNode.LoadMetadata; - childObjects.Add(new LoadStageNode(lmd, LoadStage.GetFiles)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.Mounting)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustRaw)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.AdjustStaging)); - childObjects.Add(new LoadStageNode(lmd, LoadStage.PostLoad)); - - foreach (LoadStageNode node in childObjects) - AddChildren(node, descendancy.Add(node)); - - AddToDictionaries(childObjects, descendancy); - } - - private void AddChildren(LoadStageNode loadStageNode, DescendancyList descendancy) - { - var tasks = AllProcessTasks.Where( - p => p.LoadMetadata_ID == loadStageNode.LoadMetadata.ID && p.LoadStage == loadStageNode.LoadStage) - .OrderBy(o => o.Order).ToArray(); + // update just in case we became a root filter for someone + aggregate.RevertToDatabaseState(); - foreach (var processTask in tasks) - AddChildren(processTask, descendancy.Add(processTask)); - - if (tasks.Any()) - AddToDictionaries(new HashSet(tasks), descendancy); - } - - private void AddChildren(ProcessTask procesTask, DescendancyList descendancy) - { - var args = AllProcessTasksArguments.Where( - a => a.ProcessTask_ID == procesTask.ID).ToArray(); - - if (args.Any()) - AddToDictionaries(new HashSet(args), descendancy); - } - - private void AddChildren(LoadMetadataVersionNode LoadMetadataVersionNode, DescendancyList descendancy) - { - LoadMetadataVersionNode.LoadMetadataVersions = AllLoadMetadatas.Where(lmd => lmd.RootLoadMetadata_ID == LoadMetadataVersionNode.LoadMetadata.ID).ToList(); - var childObjects = new List(); - - foreach (var lmd in LoadMetadataVersionNode.LoadMetadataVersions) - { - AddChildren(lmd, descendancy.Add(lmd), false, false, false); - childObjects.Add(lmd); - } - AddToDictionaries(new HashSet(childObjects), descendancy); - - } - - private void AddChildren(AllCataloguesUsedByLoadMetadataNode allCataloguesUsedByLoadMetadataNode, - DescendancyList descendancy) - { - var loadMetadataId = allCataloguesUsedByLoadMetadataNode.LoadMetadata.ID; - var linkedCatalogueIDs = AllLoadMetadataLinkage.Where(link => link.LoadMetadataID == loadMetadataId).Select(static link => link.CatalogueID); - var usedCatalogues = linkedCatalogueIDs.Select(catalogueId => AllCatalogues.FirstOrDefault(c => c.ID == catalogueId)).Where(static foundCatalogue => foundCatalogue is not null).ToList(); - allCataloguesUsedByLoadMetadataNode.UsedCatalogues = usedCatalogues; - var childObjects = usedCatalogues.Select(foundCatalogue => new CatalogueUsedByLoadMetadataNode(allCataloguesUsedByLoadMetadataNode.LoadMetadata, foundCatalogue)).Cast().ToHashSet(); - - AddToDictionaries(childObjects, descendancy); - } - - #endregion - - protected void AddChildren(Catalogue c, DescendancyList descendancy) - { - var childObjects = new List(); - - var catalogueAggregates = AllAggregateConfigurations.Where(a => a.Catalogue_ID == c.ID).ToArray(); - var cohortAggregates = catalogueAggregates.Where(a => a.IsCohortIdentificationAggregate).ToArray(); - var regularAggregates = catalogueAggregates.Except(cohortAggregates).ToArray(); - - //get all the CatalogueItems for this Catalogue (TryGet because Catalogue may not have any items - var cis = _catalogueToCatalogueItems.TryGetValue(c.ID, out var result) - ? result.ToArray() - : Array.Empty(); - - //tell the CatalogueItems that we are are their parent - foreach (var ci in cis) - ci.InjectKnown(c); - - // core includes project specific which basically means the same thing - var core = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Core || - ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.ProjectSpecific) - , ExtractionCategory.Core); - - c.InjectKnown(cis); - - var deprecated = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Deprecated), - ExtractionCategory.Deprecated); - var special = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.SpecialApprovalRequired), - ExtractionCategory.SpecialApprovalRequired); - var intern = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Internal), - ExtractionCategory.Internal); - var supplemental = new CatalogueItemsNode(c, - cis.Where(ci => ci.ExtractionInformation?.ExtractionCategory == ExtractionCategory.Supplemental), - ExtractionCategory.Supplemental); - var notExtractable = new CatalogueItemsNode(c, cis.Where(ci => ci.ExtractionInformation == null), null); - - AddChildren(core, descendancy.Add(core)); - childObjects.Add(core); - - foreach (var optional in new[] { deprecated, special, intern, supplemental, notExtractable }) - if (optional.CatalogueItems.Any()) - { - AddChildren(optional, descendancy.Add(optional)); - childObjects.Add(optional); - } - - //do we have any foreign key fields into this lookup table - var lookups = AllLookups.Where(l => c.CatalogueItems.Any(ci => ci.ColumnInfo_ID == l.ForeignKey_ID)).ToArray(); - - var docs = AllSupportingDocuments.Where(d => d.Catalogue_ID == c.ID).ToArray(); - var sql = AllSupportingSQL.Where(d => d.Catalogue_ID == c.ID).ToArray(); - - //if there are supporting documents or supporting sql files then add documentation node - if (docs.Any() || sql.Any()) - { - var documentationNode = new DocumentationNode(c, docs, sql); - - //add the documentations node - childObjects.Add(documentationNode); - - //record the children - AddToDictionaries(new HashSet(docs.Cast().Union(sql)), descendancy.Add(documentationNode)); - } - - if (lookups.Any()) - { - var lookupsNode = new CatalogueLookupsNode(c, lookups); - //add the documentations node - childObjects.Add(lookupsNode); - - - //record the children - AddToDictionaries(new HashSet(lookups.Select(l => new CatalogueLookupUsageNode(c, l))), - descendancy.Add(lookupsNode)); - } - - if (regularAggregates.Any()) - { - var aggregatesNode = new AggregatesNode(c, regularAggregates); - childObjects.Add(aggregatesNode); - - var nodeDescendancy = descendancy.Add(aggregatesNode); - AddToDictionaries(new HashSet(regularAggregates), nodeDescendancy); - - foreach (var regularAggregate in regularAggregates) - AddChildren(regularAggregate, nodeDescendancy.Add(regularAggregate)); - } - - //finalise - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - private void InjectCatalogueItems() - { - foreach (var ci in AllCatalogueItems) - if (_extractionInformationsByCatalogueItem.TryGetValue(ci.ID, out var ei)) - ci.InjectKnown(ei); - else - ci.InjectKnown((ExtractionInformation)null); - } - - private void AddChildren(CatalogueItemsNode node, DescendancyList descendancyList) - { - AddToDictionaries(new HashSet(node.CatalogueItems), descendancyList); - - foreach (var ci in node.CatalogueItems) - AddChildren(ci, descendancyList.Add(ci)); - } - - private void AddChildren(AggregateConfiguration aggregateConfiguration, DescendancyList descendancy) - { - var childrenObjects = new HashSet(); - - var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(aggregateConfiguration)).Cast() - .ToArray(); - - foreach (var p in parameters) - childrenObjects.Add(p); - - // show the dimensions in the tree - foreach (var dim in aggregateConfiguration.AggregateDimensions) childrenObjects.Add(dim); - - // show the axis (if any) in the tree. If there are multiple axis in this tree then that is bad but maybe the user can delete one of them to fix the situation - foreach (var axis in AllAggregateContinuousDateAxis.Where(a => - aggregateConfiguration.AggregateDimensions.Any(d => d.ID == a.AggregateDimension_ID))) - childrenObjects.Add(axis); - - //we can step into this twice, once via Catalogue children and once via CohortIdentificationConfiguration children - //if we get in via Catalogue children then descendancy will be Ignore=true we don't end up emphasising into CatalogueCollectionUI when - //really user wants to see it in CohortIdentificationCollectionUI - if (aggregateConfiguration.RootFilterContainer_ID != null) - { - var container = AllAggregateContainersDictionary[(int)aggregateConfiguration.RootFilterContainer_ID]; - - AddChildren(container, descendancy.Add(container)); - childrenObjects.Add(container); - } - - AddToDictionaries(childrenObjects, descendancy); - } + BuildAggregateFilterContainers(); - private void AddChildren(AggregateFilterContainer container, DescendancyList descendancy) - { - var childrenObjects = new List(); - - var subcontainers = _aggregateFilterManager.GetSubContainers(container); - var filters = _aggregateFilterManager.GetFilters(container); - - foreach (AggregateFilterContainer subcontainer in subcontainers) - { - //one of our children is this subcontainer - childrenObjects.Add(subcontainer); - - //but also document its children - AddChildren(subcontainer, descendancy.Add(subcontainer)); - } - - //also add the filters for the container - foreach (var f in filters) - { - // for filters add the parameters under them - AddChildren((AggregateFilter)f, descendancy.Add(f)); - childrenObjects.Add(f); - } - - //add our children to the dictionary - AddToDictionaries(new HashSet(childrenObjects), descendancy); - } - - private void AddChildren(AggregateFilter f, DescendancyList descendancy) - { - AddToDictionaries(new HashSet(AllAggregateFilterParameters.Where(p => p.AggregateFilter_ID == f.ID)), - descendancy); - } - - private void AddChildren(CatalogueItem ci, DescendancyList descendancy) - { - var childObjects = new List(); - - var ei = ci.ExtractionInformation; - if (ei != null) - { - childObjects.Add(ei); - AddChildren(ei, descendancy.Add(ei)); - } - else - { - ci.InjectKnown( - (ExtractionInformation)null); // we know the CatalogueItem has no ExtractionInformation child because it's not in the dictionary - } - - if (ci.ColumnInfo_ID.HasValue && _allColumnInfos.TryGetValue(ci.ColumnInfo_ID.Value, out var col)) - childObjects.Add(new LinkedColumnInfoNode(ci, col)); - - AddToDictionaries(new HashSet(childObjects), descendancy); - } - - private void AddChildren(ExtractionInformation extractionInformation, DescendancyList descendancy) - { - var children = new HashSet(); - - foreach (var filter in AllCatalogueFilters.Where(f => f.ExtractionInformation_ID == extractionInformation.ID)) - { - //add the filter as a child of the - children.Add(filter); - AddChildren(filter, descendancy.Add(filter)); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(ExtractionFilter filter, DescendancyList descendancy) - { - var children = new HashSet(); - var parameters = AllCatalogueParameters.Where(p => p.ExtractionFilter_ID == filter.ID).ToArray(); - var parameterSets = AllCatalogueValueSets.Where(vs => vs.ExtractionFilter_ID == filter.ID).ToArray(); - - filter.InjectKnown(parameterSets); - - foreach (var p in parameters) - children.Add(p); - - foreach (var set in parameterSets) - { - children.Add(set); - AddChildren(set, descendancy.Add(set), parameters); - } - - if (children.Any()) - AddToDictionaries(children, descendancy); - } - - private void AddChildren(ExtractionFilterParameterSet set, DescendancyList descendancy, - ExtractionFilterParameter[] filterParameters) - { - var children = new HashSet(); - - foreach (var setValue in AllCatalogueValueSetValues.Where(v => v.ExtractionFilterParameterSet_ID == set.ID)) - { - setValue.InjectKnown(filterParameters.SingleOrDefault(p => p.ID == setValue.ExtractionFilterParameter_ID)); - children.Add(setValue); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(CohortIdentificationConfiguration cic, DescendancyList descendancy) - { - var children = new HashSet(); - - //it has an associated query cache - if (cic.QueryCachingServer_ID != null) - children.Add(new QueryCacheUsedByCohortIdentificationNode(cic, - AllExternalServers.Single(s => s.ID == cic.QueryCachingServer_ID))); - - var parameters = AllAnyTableParameters.Where(p => p.IsReferenceTo(cic)).Cast().ToArray(); - foreach (var p in parameters) children.Add(p); - - //if it has a root container - if (cic.RootCohortAggregateContainer_ID != null) - { - var container = AllCohortAggregateContainers.Single(c => c.ID == cic.RootCohortAggregateContainer_ID); - AddChildren(container, descendancy.Add(container).SetBetterRouteExists()); - children.Add(container); - } - - //get the patient index tables - var joinableNode = new JoinableCollectionNode(cic, - AllJoinables.Where(j => j.CohortIdentificationConfiguration_ID == cic.ID).ToArray()); - AddChildren(joinableNode, descendancy.Add(joinableNode).SetBetterRouteExists()); - children.Add(joinableNode); - - AddToDictionaries(children, descendancy.SetBetterRouteExists()); - } - - private void AddChildren(JoinableCollectionNode joinablesNode, DescendancyList descendancy) - { - var children = new HashSet(); - - foreach (var joinable in joinablesNode.Joinables) - try - { - var agg = AllAggregateConfigurations.Single(ac => ac.ID == joinable.AggregateConfiguration_ID); - ForceAggregateNaming(agg, descendancy); - children.Add(agg); - - //it's no longer an orphan because it's in a known cic (as a patient index table) - OrphanAggregateConfigurations.Remove(agg); - - AddChildren(agg, descendancy.Add(agg)); - } - catch (Exception e) - { - throw new Exception( - $"JoinableCohortAggregateConfiguration (patient index table) object (ID={joinable.ID}) references AggregateConfiguration_ID {joinable.AggregateConfiguration_ID} but that AggregateConfiguration was not found", - e); - } - - AddToDictionaries(children, descendancy); - } - - private void AddChildren(CohortAggregateContainer container, DescendancyList descendancy) - { - //get subcontainers - var subcontainers = _cohortContainerManager.GetChildren(container).OfType().ToList(); - - //if there are subcontainers - foreach (var subcontainer in subcontainers) - AddChildren(subcontainer, descendancy.Add(subcontainer)); - - //get our configurations - var configurations = _cohortContainerManager.GetChildren(container).OfType().ToList(); - - //record the configurations children including full descendancy - foreach (var configuration in configurations) - { - ForceAggregateNaming(configuration, descendancy); - AddChildren(configuration, descendancy.Add(configuration)); - - //it's no longer an orphan because it's in a known cic - OrphanAggregateConfigurations.Remove(configuration); - } - - //all our children (containers and aggregates) - //children are all aggregates and containers at the current hierarchy level in order - var children = subcontainers.Union(configurations.Cast()).OrderBy(o => o.Order).ToList(); - - AddToDictionaries(new HashSet(children), descendancy); - } - - private void ForceAggregateNaming(AggregateConfiguration configuration, DescendancyList descendancy) - { - //configuration has the wrong name - if (!configuration.IsCohortIdentificationAggregate) - { - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs( - $"Had to fix naming of configuration '{configuration}' because it didn't start with correct cic prefix", - CheckResult.Warning)); - descendancy.Parents.OfType().Single() - .EnsureNamingConvention(configuration); - configuration.SaveToDatabase(); - } - } - - private void AddChildren(TableInfoServerNode serverNode, DescendancyList descendancy) - { - //add empty hashset - var children = new HashSet(); - - var databases = - serverNode.Tables.GroupBy( - k => k.Database ?? TableInfoDatabaseNode.NullDatabaseNode, StringComparer.CurrentCultureIgnoreCase) - .Select(g => new TableInfoDatabaseNode(g.Key, serverNode, g)); - - foreach (var db in databases) - { - children.Add(db); - AddChildren(db, descendancy.Add(db)); - } - - //now we have recorded all the children add them with descendancy - AddToDictionaries(children, descendancy); - } - - private void AddChildren(TableInfoDatabaseNode dbNode, DescendancyList descendancy) - { - //add empty hashset - var children = new HashSet(); - - foreach (var t in dbNode.Tables) - { - //record the children of the table infos (mostly column infos) - children.Add(t); - - //the all servers node=>the TableInfoServerNode => the t - AddChildren(t, descendancy.Add(t)); - } - - //now we have recorded all the children add them with descendancy - AddToDictionaries(children, descendancy); - } - - private void AddChildren(TableInfo tableInfo, DescendancyList descendancy) - { - //add empty hashset - var children = new HashSet(); - - //if the table has an identifier dump listed - if (tableInfo.IdentifierDumpServer_ID != null) - { - //if there is a dump (e.g. for dilution and dumping - not appearing in the live table) - var server = AllExternalServers.Single(s => s.ID == tableInfo.IdentifierDumpServer_ID.Value); - - children.Add(new IdentifierDumpServerUsageNode(tableInfo, server)); - } - - //get the discarded columns in this table - var discardedCols = new HashSet(AllPreLoadDiscardedColumns.Where(c => c.TableInfo_ID == tableInfo.ID)); - - //tell the column who their parent is so they don't need to look up the database - foreach (PreLoadDiscardedColumn discardedCol in discardedCols) - discardedCol.InjectKnown(tableInfo); - - //if there are discarded columns - if (discardedCols.Any()) - { - var identifierDumpNode = new PreLoadDiscardedColumnsNode(tableInfo); - - //record that the usage is a child of TableInfo - children.Add(identifierDumpNode); - - //record that the discarded columns are children of identifier dump usage node - AddToDictionaries(discardedCols, descendancy.Add(identifierDumpNode)); - } - - //if it is a table valued function - if (tableInfo.IsTableValuedFunction) - { - //that has parameters - var parameters = tableInfo.GetAllParameters(); - - foreach (var p in parameters) children.Add(p); - } - - //next add the column infos - if (TableInfosToColumnInfos.TryGetValue(tableInfo.ID, out var result)) - foreach (var c in result) - { - children.Add(c); - c.InjectKnown(tableInfo); - AddChildren(c, descendancy.Add(c).SetBetterRouteExists()); - } - - //finally add any credentials objects - if (AllDataAccessCredentialUsages.TryGetValue(tableInfo, out var nodes)) - foreach (var node in nodes) - children.Add(node); - - //now we have recorded all the children add them with descendancy via the TableInfo descendancy - AddToDictionaries(children, descendancy); - } - - private void AddChildren(ColumnInfo columnInfo, DescendancyList descendancy) - { - var lookups = AllLookups.Where(l => l.Description_ID == columnInfo.ID).ToArray(); - var joinInfos = AllJoinInfos.Where(j => j.PrimaryKey_ID == columnInfo.ID); - - var children = new HashSet(); - - foreach (var l in lookups) - children.Add(l); - - foreach (var j in joinInfos) - children.Add(j); - - if (children.Any()) - AddToDictionaries(children, descendancy); - } - - protected void AddToDictionaries(HashSet children, DescendancyList list) - { - if (list.IsEmpty) - throw new ArgumentException("DescendancyList cannot be empty", nameof(list)); - - //document that the last parent has these as children - var parent = list.Last(); - - _childDictionary.AddOrUpdate(parent, - children, (p, s) => children); - - //now document the entire parent order to reach each child object i.e. 'Root=>Grandparent=>Parent' is how you get to 'Child' - foreach (var o in children) - _descendancyDictionary.AddOrUpdate(o, list, (k, v) => HandleDescendancyCollision(k, v, list)); - - - foreach (var masquerader in children.OfType()) - { - var key = masquerader.MasqueradingAs(); - - if (!AllMasqueraders.ContainsKey(key)) - AllMasqueraders.AddOrUpdate(key, new HashSet(), (o, set) => set); - - lock (AllMasqueraders) - { - AllMasqueraders[key].Add(masquerader); - } - } - } - - private static DescendancyList HandleDescendancyCollision(object key, DescendancyList oldRoute, - DescendancyList newRoute) - { - //if the new route is the best best - if (newRoute.NewBestRoute && !oldRoute.NewBestRoute) - return newRoute; - - // If the new one is marked BetterRouteExists just throw away the new one - return newRoute.BetterRouteExists ? oldRoute : newRoute; - // If in doubt use the newest one - } - - private HashSet GetAllObjects() - { - //anything which has children or is a child of someone else (distinct because HashSet) - return new HashSet(_childDictionary.SelectMany(kvp => kvp.Value).Union(_childDictionary.Keys)); - } - - public virtual object[] GetChildren(object model) - { - lock (WriteLock) - { - //if we have a record of any children in the child dictionary for the parent model object - if (_childDictionary.TryGetValue(model, out var cached)) - return cached.OrderBy(static o => o.ToString()).ToArray(); - - return model switch - { - //if they want the children of a Pipeline (which we don't track) just serve the components - Pipeline p => p.PipelineComponents.ToArray(), - //if they want the children of a PipelineComponent (which we don't track) just serve the arguments - PipelineComponent pc => pc.PipelineComponentArguments.ToArray(), - _ => Array.Empty() - }; - } - } - - public IEnumerable GetAllObjects(Type type, bool unwrapMasqueraders) - { - lock (WriteLock) - { - //things that are a match on Type but not IMasqueradeAs - var exactMatches = GetAllSearchables().Keys.Where(t => t is not IMasqueradeAs).Where(type.IsInstanceOfType); - - //Union the unwrapped masqueraders - return unwrapMasqueraders - ? exactMatches.Union( - AllMasqueraders - .Select(kvp => kvp.Key) - .OfType() - .Where(type.IsInstanceOfType)) - .Distinct() - : exactMatches; - } - } - - public DescendancyList GetDescendancyListIfAnyFor(object model) - { - lock (WriteLock) - { - return _descendancyDictionary.GetValueOrDefault(model); - } - } - - - public object GetRootObjectOrSelf(object objectToEmphasise) - { - lock (WriteLock) - { - var descendancy = GetDescendancyListIfAnyFor(objectToEmphasise); - - return descendancy != null && descendancy.Parents.Any() ? descendancy.Parents[0] : objectToEmphasise; - } - } - - - public virtual Dictionary GetAllSearchables() - { - lock (WriteLock) - { - var toReturn = new Dictionary(); - - foreach (var kvp in _descendancyDictionary.Where(kvp => kvp.Key is IMapsDirectlyToDatabaseTable)) - toReturn.Add((IMapsDirectlyToDatabaseTable)kvp.Key, kvp.Value); - - return toReturn; - } - } - - public IEnumerable GetAllChildrenRecursively(object o) - { - lock (WriteLock) - { - var toReturn = new List(); - - foreach (var child in GetChildren(o)) - { - toReturn.Add(child); - toReturn.AddRange(GetAllChildrenRecursively(child)); - } - - return toReturn; - } - } - - /// - /// Asks all plugins to provide the child objects for every object we have found so far. This method is recursive, call it with null the first time to use all objects. It will then - /// call itself with all the new objects that were sent back by the plugin (so that new objects found can still have children). - /// - /// - protected void GetPluginChildren(HashSet objectsToAskAbout = null) - { - lock (WriteLock) - { - var newObjectsFound = new HashSet(); - - var sw = new Stopwatch(); - - var providers = _pluginChildProviders.Except(_blockedPlugins).ToArray(); - - //for every object found so far - if (providers.Any()) - foreach (var o in objectsToAskAbout ?? GetAllObjects()) - //for every plugin loaded (that is not forbidlisted) - foreach (var plugin in providers) - //ask about the children - try - { - sw.Restart(); - //otherwise ask plugin what its children are - var pluginChildren = plugin.GetChildren(o); - - //if the plugin takes too long to respond we need to stop - if (sw.ElapsedMilliseconds > 1000) - { - _blockedPlugins.Add(plugin); - throw new Exception( - $"Plugin '{plugin}' was forbidlisted for taking too long to respond to GetChildren(o) where o was a '{o.GetType().Name}' ('{o}')"); - } - - //it has children - if (pluginChildren != null && pluginChildren.Any()) - { - //get the descendancy of the parent - var parentDescendancy = GetDescendancyListIfAnyFor(o); - var newDescendancy = parentDescendancy == null - ? new DescendancyList(new[] { o }) - : //if the parent is a root level object start a new descendancy list from it - parentDescendancy - .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one - newDescendancy = - parentDescendancy - .Add(o); //otherwise keep going down, returns a new DescendancyList so doesn't corrupt the dictionary one - - //record that - foreach (var pluginChild in pluginChildren) - { - //if the parent didn't have any children before - if (!_childDictionary.ContainsKey(o)) - _childDictionary.AddOrUpdate(o, new HashSet(), - (o1, set) => set); //it does now - - - //add us to the parent objects child collection - _childDictionary[o].Add(pluginChild); - - //add to the child collection of the parent object kvp.Key - _descendancyDictionary.AddOrUpdate(pluginChild, newDescendancy, - (s, e) => newDescendancy); - - //we have found a new object so we must ask other plugins about it (chances are a plugin will have a whole tree of sub objects) - newObjectsFound.Add(pluginChild); - } - } - } - catch (Exception e) - { - _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail, e)); - } - - if (newObjectsFound.Any()) - GetPluginChildren(newObjectsFound); - } - } - - public IEnumerable GetMasqueradersOf(object o) - { - lock (WriteLock) - { - return AllMasqueraders.TryGetValue(o, out var result) ? result : Array.Empty(); - } - } - - protected T[] GetAllObjects(IRepository repository) where T : IMapsDirectlyToDatabaseTable - { - lock (WriteLock) - { - return repository.GetAllObjects(); - } - } - - - protected void AddToReturnSearchablesWithNoDecendancy( - Dictionary toReturn, - IEnumerable toAdd) - { - lock (WriteLock) - { - foreach (var m in toAdd) - toReturn.Add(m, null); - } - } - - public virtual void UpdateTo(ICoreChildProvider other) - { - ArgumentNullException.ThrowIfNull(other); - - if (other is not CatalogueChildProvider otherCat) - throw new NotSupportedException( - $"Did not know how to UpdateTo ICoreChildProvider of type {other.GetType().Name}"); - - AllLoadMetadatas = otherCat.AllLoadMetadatas; - AllProcessTasks = otherCat.AllProcessTasks; - AllProcessTasksArguments = otherCat.AllProcessTasksArguments; - AllLoadProgresses = otherCat.AllLoadProgresses; - AllCacheProgresses = otherCat.AllCacheProgresses; - AllPermissionWindows = otherCat.AllPermissionWindows; - AllCatalogues = otherCat.AllCatalogues; - AllCataloguesDictionary = otherCat.AllCataloguesDictionary; - AllSupportingDocuments = otherCat.AllSupportingDocuments; - AllSupportingSQL = otherCat.AllSupportingSQL; - _childDictionary = otherCat._childDictionary; - _descendancyDictionary = otherCat._descendancyDictionary; - _catalogueToCatalogueItems = otherCat._catalogueToCatalogueItems; - AllCatalogueItemsDictionary = otherCat.AllCatalogueItemsDictionary; - _allColumnInfos = otherCat._allColumnInfos; - AllAggregateConfigurations = otherCat.AllAggregateConfigurations; - AllAggregateDimensions = otherCat.AllAggregateDimensions; - AllAggregateContinuousDateAxis = otherCat.AllAggregateContinuousDateAxis; - AllRDMPRemotesNode = otherCat.AllRDMPRemotesNode; - AllRemoteRDMPs = otherCat.AllRemoteRDMPs; - AllDashboardsNode = otherCat.AllDashboardsNode; - AllDashboards = otherCat.AllDashboards; - AllObjectSharingNode = otherCat.AllObjectSharingNode; - AllImports = otherCat.AllImports; - AllExports = otherCat.AllExports; - AllStandardRegexesNode = otherCat.AllStandardRegexesNode; - AllPipelinesNode = otherCat.AllPipelinesNode; - OtherPipelinesNode = otherCat.OtherPipelinesNode; - AllPipelines = otherCat.AllPipelines; - AllPipelineComponents = otherCat.AllPipelineComponents; - AllPipelineComponentsArguments = otherCat.AllPipelineComponentsArguments; - AllStandardRegexes = otherCat.AllStandardRegexes; - AllANOTablesNode = otherCat.AllANOTablesNode; - AllANOTables = otherCat.AllANOTables; - AllExternalServers = otherCat.AllExternalServers; - AllServers = otherCat.AllServers; - AllTableInfos = otherCat.AllTableInfos; - AllDataAccessCredentialsNode = otherCat.AllDataAccessCredentialsNode; - AllExternalServersNode = otherCat.AllExternalServersNode; - AllServersNode = otherCat.AllServersNode; - AllDataAccessCredentials = otherCat.AllDataAccessCredentials; - AllDataAccessCredentialUsages = otherCat.AllDataAccessCredentialUsages; - TableInfosToColumnInfos = otherCat.TableInfosToColumnInfos; - AllColumnInfos = otherCat.AllColumnInfos; - AllPreLoadDiscardedColumns = otherCat.AllPreLoadDiscardedColumns; - AllLookups = otherCat.AllLookups; - AllJoinInfos = otherCat.AllJoinInfos; - AllAnyTableParameters = otherCat.AllAnyTableParameters; - AllMasqueraders = otherCat.AllMasqueraders; - AllExtractionInformationsDictionary = otherCat.AllExtractionInformationsDictionary; - _pluginChildProviders = otherCat._pluginChildProviders; - AllPermissionWindowsNode = otherCat.AllPermissionWindowsNode; - LoadMetadataRootFolder = otherCat.LoadMetadataRootFolder; - CatalogueRootFolder = otherCat.CatalogueRootFolder; - CohortIdentificationConfigurationRootFolder = otherCat.CohortIdentificationConfigurationRootFolder; - AllConnectionStringKeywordsNode = otherCat.AllConnectionStringKeywordsNode; - AllConnectionStringKeywords = otherCat.AllConnectionStringKeywords; - AllAggregateContainersDictionary = otherCat.AllAggregateContainersDictionary; - AllAggregateFilters = otherCat.AllAggregateFilters; - AllAggregateFilterParameters = otherCat.AllAggregateFilterParameters; - AllCohortIdentificationConfigurations = otherCat.AllCohortIdentificationConfigurations; - AllCohortAggregateContainers = otherCat.AllCohortAggregateContainers; - AllJoinables = otherCat.AllJoinables; - AllJoinUses = otherCat.AllJoinUses; - AllGovernanceNode = otherCat.AllGovernanceNode; - AllGovernancePeriods = otherCat.AllGovernancePeriods; - AllGovernanceDocuments = otherCat.AllGovernanceDocuments; - GovernanceCoverage = otherCat.GovernanceCoverage; - AllJoinableCohortAggregateConfigurationUse = otherCat.AllJoinableCohortAggregateConfigurationUse; - AllPluginsNode = otherCat.AllPluginsNode; - PipelineUseCases = otherCat.PipelineUseCases; - OrphanAggregateConfigurationsNode = otherCat.OrphanAggregateConfigurationsNode; - TemplateAggregateConfigurationsNode = otherCat.TemplateAggregateConfigurationsNode; - AllCatalogueParameters = otherCat.AllCatalogueParameters; - AllCatalogueValueSets = otherCat.AllCatalogueValueSets; - AllCatalogueValueSetValues = otherCat.AllCatalogueValueSetValues; - OrphanAggregateConfigurations = otherCat.OrphanAggregateConfigurations; - } - - public virtual bool SelectiveRefresh(IMapsDirectlyToDatabaseTable databaseEntity) - { - ProgressStopwatch.Restart(); - - return databaseEntity switch - { - AggregateFilterParameter afp => SelectiveRefresh(afp.AggregateFilter), - AggregateFilter af => SelectiveRefresh(af), - AggregateFilterContainer afc => SelectiveRefresh(afc), - CohortAggregateContainer cac => SelectiveRefresh(cac), - ExtractionInformation ei => SelectiveRefresh(ei), - CatalogueItem ci => SelectiveRefresh(ci), - _ => false - }; - } - - - public bool SelectiveRefresh(CatalogueItem ci) - { - var descendancy = GetDescendancyListIfAnyFor(ci.Catalogue); - if (descendancy == null) return false; - - FetchCatalogueItems(); - FetchExtractionInformations(); - AddChildren(ci.Catalogue, descendancy.Add(ci.Catalogue)); - return true; - } - - public bool SelectiveRefresh(ExtractionInformation ei) - { - var descendancy = GetDescendancyListIfAnyFor(ei); - var cata = descendancy?.Parents.OfType().LastOrDefault() ?? ei.CatalogueItem.Catalogue; - - if (cata == null) - return false; - - var cataDescendancy = GetDescendancyListIfAnyFor(cata); - - if (cataDescendancy == null) - return false; - - FetchCatalogueItems(); - - foreach (var ci in AllCatalogueItems.Where(ci => ci.ID == ei.CatalogueItem_ID)) ci.ClearAllInjections(); - - // property changes or deleting the ExtractionInformation - FetchExtractionInformations(); - - // refresh the Catalogue - AddChildren(cata, cataDescendancy.Add(cata)); - return true; - } - - public bool SelectiveRefresh(CohortAggregateContainer container) - { - var parentContainer = container.GetParentContainerIfAny(); - if (parentContainer != null) - { - var descendancy = GetDescendancyListIfAnyFor(parentContainer); - - if (descendancy != null) - { - BuildAggregateConfigurations(); - - BuildCohortCohortAggregateContainers(); - AddChildren(parentContainer, descendancy.Add(parentContainer)); - return true; - } - } - - var cic = container.GetCohortIdentificationConfiguration(); - - if (cic != null) - { - var descendancy = GetDescendancyListIfAnyFor(cic); - - if (descendancy != null) - { - BuildAggregateConfigurations(); - BuildCohortCohortAggregateContainers(); - AddChildren(cic, descendancy.Add(cic)); - return true; - } - } - - return false; - } - - public bool SelectiveRefresh(AggregateFilter f) - { - var knownContainer = GetDescendancyListIfAnyFor(f.FilterContainer); - if (knownContainer == null) return false; - - BuildAggregateFilterContainers(); - AddChildren((AggregateFilterContainer)f.FilterContainer, knownContainer.Add(f.FilterContainer)); - return true; - } - - public bool SelectiveRefresh(AggregateFilterContainer container) - { - var aggregate = container.GetAggregate(); - - if (aggregate == null) return false; - - var descendancy = GetDescendancyListIfAnyFor(aggregate); - - if (descendancy == null) return false; - - // update just in case we became a root filter for someone - aggregate.RevertToDatabaseState(); - - BuildAggregateFilterContainers(); - - AddChildren(aggregate, descendancy.Add(aggregate)); - return true; - } + AddChildren(aggregate, descendancy.Add(aggregate)); + return true; + } } \ No newline at end of file diff --git a/Rdmp.Core/Providers/CatalogueProblemProvider.cs b/Rdmp.Core/Providers/CatalogueProblemProvider.cs index 3c4e5217ea..2f6731e291 100644 --- a/Rdmp.Core/Providers/CatalogueProblemProvider.cs +++ b/Rdmp.Core/Providers/CatalogueProblemProvider.cs @@ -232,7 +232,7 @@ private string DescribeProblem(AllGovernanceNode allGovernanceNode) } var expiredCatalogues = expiredCatalogueIds.Select(id => _childProvider.AllCataloguesDictionary[id]) - .Where(c => !c.IsDeprecated /* || c.IsColdStorage || c.IsInternal*/).ToArray(); + .Where(c => !c.IsDeprecated /* || c.IsInternal*/).ToArray(); if (expiredCatalogues.Any()) return diff --git a/Rdmp.Core/Providers/DataExportChildProvider.cs b/Rdmp.Core/Providers/DataExportChildProvider.cs index 8aa30541f3..1ba9c3d71e 100644 --- a/Rdmp.Core/Providers/DataExportChildProvider.cs +++ b/Rdmp.Core/Providers/DataExportChildProvider.cs @@ -351,6 +351,23 @@ private void AddChildren(ProjectCohortsNode projectCohortsNode, DescendancyList children.Add(associatedCohortConfigurations); AddChildren(associatedCohortConfigurations, descendancy.Add(associatedCohortConfigurations)); + + var associatedTemplatesNode = new AssociatedCohortIdentificationTemplatesNode(projectCohortsNode.Project); + children.Add(associatedTemplatesNode); + AddChildren(associatedTemplatesNode, descendancy.Add(associatedTemplatesNode)); + + AddToDictionaries(children, descendancy); + } + + private void AddChildren(AssociatedCohortIdentificationTemplatesNode associatedCohortIdentificationTemplatesNode, DescendancyList descendancy) + { + var children = new HashSet(); + var associatedCohorts = associatedCohortIdentificationTemplatesNode.Project.GetAssociatedTemplateCohortIdentificationConfigurations(); + foreach (var cohort in associatedCohorts) + { + children.Add(cohort); + } + AddToDictionaries(children, descendancy); } @@ -390,6 +407,7 @@ private void AddChildren(ProjectCohortIdentificationConfigurationAssociationsNod assoc.Project_ID == projectCiCsNode.Project.ID)) { var matchingCic = AllCohortIdentificationConfigurations.SingleOrDefault(cic => + cic.ID == association.CohortIdentificationConfiguration_ID) ?? AllTemplateCohortIdentificationConfigurations.SingleOrDefault(cic => cic.ID == association.CohortIdentificationConfiguration_ID); if (matchingCic == null) @@ -830,6 +848,7 @@ public override void UpdateTo(ICoreChildProvider other) AllDatasetsNode = dxOther.AllDatasetsNode; AllRegexRedactionConfigurations = dxOther.AllRegexRedactionConfigurations; AllRegexRedactionConfigurationsNode = dxOther.AllRegexRedactionConfigurationsNode; + AllDatasetProviderConfigurationsNode = dxOther.AllDatasetProviderConfigurationsNode; } } diff --git a/Rdmp.Core/Providers/ICoreChildProvider.cs b/Rdmp.Core/Providers/ICoreChildProvider.cs index 911b02a738..0a7929dbf0 100644 --- a/Rdmp.Core/Providers/ICoreChildProvider.cs +++ b/Rdmp.Core/Providers/ICoreChildProvider.cs @@ -35,6 +35,7 @@ public interface ICoreChildProvider : IChildProvider { JoinInfo[] AllJoinInfos { get; } LoadMetadata[] AllLoadMetadatas { get; } + LoadMetadataCatalogueLinkage[] AllLoadMetadataCatalogueLinkages { get; } TableInfoServerNode[] AllServers { get; } TableInfo[] AllTableInfos { get; } Dictionary> TableInfosToColumnInfos { get; } @@ -44,12 +45,14 @@ public interface ICoreChildProvider : IChildProvider JoinableCohortAggregateConfigurationUse[] AllJoinUses { get; set; } FolderNode CatalogueRootFolder { get; } - FolderNode DatasetRootFolder { get; } + FolderNode DatasetRootFolder { get; } FolderNode LoadMetadataRootFolder { get; } FolderNode CohortIdentificationConfigurationRootFolder { get; } FolderNode CohortIdentificationConfigurationRootFolderWithoutVersionedConfigurations { get; } + + AllTemplateCohortIdentificationConfigurationsNode AllTemplateCohortIdentificationConfigurationsNode { get; } Catalogue[] AllCatalogues { get; } - Curation.Data.Dataset[] AllDatasets { get; } + Curation.Data.Datasets.Dataset[] AllDatasets { get; } Dictionary AllCataloguesDictionary { get; } ExternalDatabaseServer[] AllExternalServers { get; } @@ -84,6 +87,7 @@ public interface ICoreChildProvider : IChildProvider AllRegexRedactionConfigurationsNode AllRegexRedactionConfigurationsNode { get; } + AllDatasetProviderConfigurationsNode AllDatasetProviderConfigurationsNode { get; } AllObjectSharingNode AllObjectSharingNode { get; } ObjectImport[] AllImports { get; } ObjectExport[] AllExports { get; } diff --git a/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs b/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs new file mode 100644 index 0000000000..e37bcdc7f0 --- /dev/null +++ b/Rdmp.Core/Providers/Nodes/AllDatasetProviderConfigurationsNode.cs @@ -0,0 +1,6 @@ +namespace Rdmp.Core.Providers.Nodes; + +public class AllDatasetProviderConfigurationsNode : SingletonNode +{ + public AllDatasetProviderConfigurationsNode() : base("Dataset Provider Configurations") { } +} \ No newline at end of file diff --git a/Rdmp.Core/Providers/Nodes/CohortNodes/AllTemplateCohortIdentificationConfigurationsNode.cs b/Rdmp.Core/Providers/Nodes/CohortNodes/AllTemplateCohortIdentificationConfigurationsNode.cs new file mode 100644 index 0000000000..ed40f2318a --- /dev/null +++ b/Rdmp.Core/Providers/Nodes/CohortNodes/AllTemplateCohortIdentificationConfigurationsNode.cs @@ -0,0 +1,16 @@ +using Rdmp.Core.DataExport.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Providers.Nodes.CohortNodes +{ + public class AllTemplateCohortIdentificationConfigurationsNode : SingletonNode + { + public AllTemplateCohortIdentificationConfigurationsNode() : base("Template Cohort Configurations") + { + } + } +} diff --git a/Rdmp.Core/Providers/Nodes/CohortNodes/AssociatedCohortIdentificationTemplatesNode.cs b/Rdmp.Core/Providers/Nodes/CohortNodes/AssociatedCohortIdentificationTemplatesNode.cs new file mode 100644 index 0000000000..a9425b52da --- /dev/null +++ b/Rdmp.Core/Providers/Nodes/CohortNodes/AssociatedCohortIdentificationTemplatesNode.cs @@ -0,0 +1,40 @@ +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.DataExport.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Providers.Nodes.CohortNodes +{ + public class AssociatedCohortIdentificationTemplatesNode:Node, IOrderable + { + public Project Project { get; set; } + + public AssociatedCohortIdentificationTemplatesNode(Project project) + { + Project = project; + } + public override string ToString() => "Associated Cohort ConfigurationTemplates"; + + protected bool Equals(AssociatedCohortIdentificationTemplatesNode other) => + Equals(Project, other.Project); + + public override bool Equals(object obj) + { + if (obj is null) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != GetType()) return false; + return Equals((AssociatedCohortIdentificationTemplatesNode)obj); + } + + public override int GetHashCode() => Project != null ? Project.GetHashCode() : 0; + + public int Order + { + get => 1; + set { } + } + } +} diff --git a/Rdmp.Core/Providers/SearchablesMatchScorer.cs b/Rdmp.Core/Providers/SearchablesMatchScorer.cs index 6f7183cba5..4f603bbf55 100644 --- a/Rdmp.Core/Providers/SearchablesMatchScorer.cs +++ b/Rdmp.Core/Providers/SearchablesMatchScorer.cs @@ -34,9 +34,7 @@ public class SearchablesMatchScorer private readonly bool _scoreZeroForCohortAggregateContainers; private bool _showInternalCatalogues = true; private bool _showDeprecatedCatalogues = true; - private bool _showColdStorageCatalogues = true; private bool _showProjectSpecificCatalogues = true; - private bool _showNonExtractableCatalogues = true; /// @@ -175,9 +173,7 @@ private void SetupRespectUserSettings() { _showInternalCatalogues = !RespectUserSettings || UserSettings.ShowInternalCatalogues; _showDeprecatedCatalogues = !RespectUserSettings || UserSettings.ShowDeprecatedCatalogues; - _showColdStorageCatalogues = !RespectUserSettings || UserSettings.ShowColdStorageCatalogues; _showProjectSpecificCatalogues = !RespectUserSettings || UserSettings.ShowProjectSpecificCatalogues; - _showNonExtractableCatalogues = !RespectUserSettings || UserSettings.ShowNonExtractableCatalogues; } private int _ScoreMatches(KeyValuePair kvp, List regexes, @@ -277,8 +273,8 @@ private int _ScoreMatches(KeyValuePair /// private bool ScoreZeroBecauseOfUserSettings(KeyValuePair kvp) => - !Filter(kvp.Key, kvp.Value, _showInternalCatalogues, _showDeprecatedCatalogues, _showColdStorageCatalogues, - _showProjectSpecificCatalogues, _showNonExtractableCatalogues); + !Filter(kvp.Key, kvp.Value, _showInternalCatalogues, _showDeprecatedCatalogues, + _showProjectSpecificCatalogues); private static Catalogue GetCatalogueIfAnyInDescendancy( KeyValuePair kvp) @@ -326,12 +322,10 @@ private static int MatchCount(List regexes, string str) /// /// /// - /// /// - /// /// True if the item should be shown to the user based on filters public static bool Filter(object modelObject, DescendancyList descendancy, bool includeInternal, - bool includeDeprecated, bool includeColdStorage, bool includeProjectSpecific, bool includeNonExtractable) + bool includeDeprecated, bool includeProjectSpecific) { //doesn't relate to us... if (modelObject is not ICatalogue cata) @@ -349,16 +343,12 @@ public static bool Filter(object modelObject, DescendancyList descendancy, bool } var isProjectSpecific = cata.IsProjectSpecific(null); - var isExtractable = cata.GetExtractabilityStatus(null) != null && - cata.GetExtractabilityStatus(null).IsExtractable; - - return (isExtractable && !cata.IsColdStorageDataset && !cata.IsDeprecated && !cata.IsInternalDataset && - !isProjectSpecific) || - (includeColdStorage && cata.IsColdStorageDataset) || - (includeDeprecated && cata.IsDeprecated) || - (includeInternal && cata.IsInternalDataset) || - (includeProjectSpecific && isProjectSpecific) || - (includeNonExtractable && !isExtractable); + + var isVisible = !cata.IsDeprecated && !cata.IsInternalDataset && !isProjectSpecific; + var showDeprecated = includeDeprecated && cata.IsDeprecated; + var showInternal = includeInternal && cata.IsInternalDataset; + var showProjectSpecific = includeProjectSpecific && isProjectSpecific; + return isVisible || showDeprecated || showInternal || showProjectSpecific; } /// diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 85c918ed5f..7c18b37136 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -256,6 +256,8 @@ + + diff --git a/Rdmp.Core/Reports/DitaCatalogueExtractor.cs b/Rdmp.Core/Reports/DitaCatalogueExtractor.cs index b615ebdf07..6e09386165 100644 --- a/Rdmp.Core/Reports/DitaCatalogueExtractor.cs +++ b/Rdmp.Core/Reports/DitaCatalogueExtractor.cs @@ -72,7 +72,7 @@ public void Extract(IDataLoadEventListener listener) //get all the catalogues then sort them alphabetically var catas = new List(_repository.GetAllObjects() - .Where(c => !(c.IsDeprecated || c.IsInternalDataset || c.IsColdStorageDataset))); + .Where(c => !(c.IsDeprecated || c.IsInternalDataset))); catas.Sort(); var sw = Stopwatch.StartNew(); @@ -270,7 +270,7 @@ private void GenerateIntroductionFile(string filename) /// public void Check(ICheckNotifier notifier) { - var catas = _repository.GetAllObjects().Where(c => !c.IsInternalDataset && !c.IsColdStorageDataset) + var catas = _repository.GetAllObjects().Where(c => !c.IsInternalDataset) .ToArray(); //Catalogues with no acronyms diff --git a/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs new file mode 100644 index 0000000000..e140e96c3f --- /dev/null +++ b/Rdmp.Core/Reports/ExtractionTime/DatasetVariableReportGenerator.cs @@ -0,0 +1,85 @@ +using CommandLine; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.DataExport.DataExtraction.Pipeline; +using Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; +using Rdmp.Core.QueryBuilding; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace Rdmp.Core.Reports.ExtractionTime +{ + class DatasetVariableReportGenerator + { + private readonly ICatalogue _catalogue; + private readonly IExecuteDatasetExtractionDestination _destination; + private readonly List _columnsToExtract; + private readonly List _releaseSubs; + + + public DatasetVariableReportGenerator(ExtractionPipelineUseCase executer) { + _destination = executer.Destination; + _catalogue = executer.Source.Request.Catalogue; + _columnsToExtract = executer.Source.Request.ColumnsToExtract.Select(c => c.Cast()).ToList(); + _releaseSubs = executer.Source.Request.ReleaseIdentifierSubstitutions; + } + + public void GenerateDatasetVariableReport() + { + var csv = new StringBuilder(); + WriteHeaders(csv); + foreach(var column in _columnsToExtract) + { + WriteColumn(column, csv); + } + foreach (var sub in _releaseSubs) + { + WriteReleaseSubs(sub, csv); + } + File.WriteAllText(new FileInfo(Path.Join(_destination.DirectoryPopulated.FullName, + $"{_destination.GetFilename()}Variables.csv")).ToString(), csv.ToString()); + } + + + private string LookupKeyNameGenerator(ColumnInfo columnInfo) + { + var split = columnInfo.GetFullyQualifiedName().Split('.'); + return $"{split[^2]}.{split[^1]}"; + } + private string LookupStringGenerator(Lookup lookup) + { + return $"{LookupKeyNameGenerator(lookup.ForeignKey)} = {LookupKeyNameGenerator(lookup.PrimaryKey)}"; + } + + private void WriteColumn(ExtractableColumn column, StringBuilder sb) + { + if (_catalogue.CatalogueItems.Length == 0) return; + var catalogueItem = _catalogue.CatalogueItems.FirstOrDefault(c => c.ColumnInfo_ID == column.ColumnInfo.ID); + if (catalogueItem is null || catalogueItem.ExtractionInformation is null) return; + bool isNull = !catalogueItem.ExtractionInformation.IsPrimaryKey; + bool isIdentifier = catalogueItem.ExtractionInformation.IsExtractionIdentifier; + var lookups = _catalogue.CatalogueRepository.GetAllObjectsWhere("ForeignKey_ID",column.ColumnInfo.ID); + var lookupString = ""; + if (lookups.Length != 0) lookupString = string.Join(';', lookups.Select(l => LookupStringGenerator(l))); + sb.AppendLine($"\"{column.GetRuntimeName()}\",\"{column.ColumnInfo.Data_type}\",{isNull},\"{catalogueItem.Description}\",{isIdentifier},{lookups.Length != 0},{lookupString}"); + } + + private void WriteReleaseSubs(ReleaseIdentifierSubstitution releaseIdentifierSubstitution,StringBuilder sb) + { + var column = releaseIdentifierSubstitution.ColumnInfo; + var catalogueItem = _catalogue.CatalogueItems.Where(c => c.ColumnInfo_ID == column.ID).First(); + var lookups = _catalogue.CatalogueRepository.GetAllObjectsWhere("ForeignKey_ID", column.ID); + var lookupString = ""; + if (lookups.Length != 0) lookupString = string.Join(';', lookups.Select(l => LookupStringGenerator(l))); + sb.AppendLine($"\"{releaseIdentifierSubstitution.Alias}\",\"{column.Data_type}\",{false},\"{catalogueItem.Description}\",{releaseIdentifierSubstitution.IsExtractionIdentifier},{lookups.Any()},{lookupString}"); + + } + + private static void WriteHeaders(StringBuilder sb) + { + sb.AppendLine("Variable Name, Type, Null possible(Y/N),Description,Identifier,HasLookups,Lookups"); + } + } +} diff --git a/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs b/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs index 0ccb871060..de05db936a 100644 --- a/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs +++ b/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs @@ -162,7 +162,7 @@ private void CreateCohortDetailsTable(XWPFDocument document) } [NotNull] - private string getDOI([NotNull] Curation.Data.Dataset ds) + private string getDOI([NotNull] Curation.Data.Datasets.Dataset ds) { return !string.IsNullOrWhiteSpace(ds.DigitalObjectIdentifier) ? $" (DOI: {ds.DigitalObjectIdentifier})" : ""; } @@ -188,7 +188,7 @@ private void CreateFileSummary(XWPFDocument document) SetTableCell(table, tableLine, 0, extractableDataset.ToString()); var linkedDatasets = extractableDataset.Catalogue.CatalogueItems.Select(static c => c.ColumnInfo).Where(ci => ci.Dataset_ID != null).Distinct().Select(ci => ci.Dataset_ID); - var datasets = _repository.CatalogueRepository.GetAllObjects().Where(d => linkedDatasets.Contains(d.ID)).ToList(); + var datasets = _repository.CatalogueRepository.GetAllObjects().Where(d => linkedDatasets.Contains(d.ID)).ToList(); var datasetString = string.Join("",datasets.Select(ds=> $"{ds.Name} {getDOI(ds)}, {Environment.NewLine}")); SetTableCell(table, tableLine, 1, result.FiltersUsed); SetTableCell(table, tableLine, 2, filename); diff --git a/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs b/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs index 45806e2f09..cc2384d162 100644 --- a/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs +++ b/Rdmp.Core/Reports/ExtractionTime/WordDataWriter.cs @@ -57,7 +57,7 @@ public WordDataWriter(ExtractionPipelineUseCase executer) [NotNull] - private static string GetDoi([NotNull] Curation.Data.Dataset ds) + private static string GetDoi([NotNull] Curation.Data.Datasets.Dataset ds) { return !string.IsNullOrWhiteSpace(ds.DigitalObjectIdentifier) ? $" (DOI: {ds.DigitalObjectIdentifier})" : ""; } @@ -163,7 +163,7 @@ public void GenerateWordFile() if (foundDatasets.Count > 0) { - var datasets = Executer.Source.Request.Catalogue.Repository.GetAllObjects().ToList(); + var datasets = Executer.Source.Request.Catalogue.Repository.GetAllObjects().ToList(); var datasetString = string.Join(", ", foundDatasets diff --git a/Rdmp.Core/Reports/GovernanceReport.cs b/Rdmp.Core/Reports/GovernanceReport.cs index d632a538c3..9d0c271bfd 100644 --- a/Rdmp.Core/Reports/GovernanceReport.cs +++ b/Rdmp.Core/Reports/GovernanceReport.cs @@ -61,7 +61,7 @@ public void GenerateReport() .Where(c => c.CatalogueItems.Any(ci => ci.ExtractionInformation != null)) .OrderBy(c => c.Name)) { - if (catalogue.IsDeprecated || catalogue.IsColdStorageDataset || catalogue.IsInternalDataset) + if (catalogue.IsDeprecated || catalogue.IsInternalDataset) continue; //get active governances diff --git a/Rdmp.Core/Repositories/DataExportRepository.cs b/Rdmp.Core/Repositories/DataExportRepository.cs index c2c9705f87..4ae7b264c3 100644 --- a/Rdmp.Core/Repositories/DataExportRepository.cs +++ b/Rdmp.Core/Repositories/DataExportRepository.cs @@ -99,7 +99,9 @@ protected override IMapsDirectlyToDatabaseTable ConstructEntity(Type t, DbDataRe public CatalogueExtractabilityStatus GetExtractabilityStatus(ICatalogue c) { var eds = GetAllObjectsWithParent(c).ToList(); - return eds.Count == 0 ? new CatalogueExtractabilityStatus(false, false) : new CatalogueExtractabilityStatus(true, eds.Count > 1 ? true : eds.First().Projects.Any()); + var isExtractable = !c.IsInternalDataset; + var extractabilityStatus = new CatalogueExtractabilityStatus(isExtractable, eds.Count != 0 && eds.First().Projects.Any()); + return extractabilityStatus; } public ISelectedDataSets[] GetSelectedDatasetsWithNoExtractionIdentifiers() => diff --git a/Rdmp.Core/Repositories/Managers/PasswordEncryptionKeyLocation.cs b/Rdmp.Core/Repositories/Managers/PasswordEncryptionKeyLocation.cs index fc151723b5..7fbb20189e 100644 --- a/Rdmp.Core/Repositories/Managers/PasswordEncryptionKeyLocation.cs +++ b/Rdmp.Core/Repositories/Managers/PasswordEncryptionKeyLocation.cs @@ -65,7 +65,7 @@ private string GetKeyFileLocationImpl() public string OpenKeyFile() { var location = GetKeyFileLocation(); - return location is null ? null : File.ReadAllText(location); + return location is null ? null : File.ReadAllText($@"{location}"); } private static void DeserializeFromLocation(string keyLocation) @@ -74,7 +74,7 @@ private static void DeserializeFromLocation(string keyLocation) return; try { - new RSACryptoServiceProvider().FromXmlString(File.ReadAllText(keyLocation)); + new RSACryptoServiceProvider().FromXmlString(File.ReadAllText($@"{keyLocation}")); } catch (Exception ex) { @@ -129,8 +129,8 @@ public FileInfo CreateNewKeyFile(string path) public void ChangeLocation(string newLocation) { ClearAllInjections(); - - if (!File.Exists(newLocation)) + + if (!File.Exists($@"{newLocation}")) throw new FileNotFoundException($"Could not find key file at:{newLocation}"); //confirms that it is accessible and deserializable diff --git a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs index 067881100a..af1e8be113 100644 --- a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs +++ b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs @@ -27,7 +27,11 @@ public static class UserSettings throw new NotImplementedException( "Isolated Storage does not work in this environment..."); - + public static bool ShowFlatLists + { + get => AppSettings.GetValueOrDefault("ShowFlatLists", false); + set => AppSettings.AddOrUpdateValue("ShowFlatLists", value); + } public static bool UseLocalFileSystem { get => AppSettings.GetValueOrDefault("UseLocalFileSystem", false); @@ -195,6 +199,12 @@ public static string ExtractionWebhookUrl set => AppSettings.AddOrUpdateValue("ExtractionWebhookUrl", value); } + public static bool DefaultLogViewFlat + { + get => AppSettings.GetValueOrDefault("DefaultLogViewFlat", false); + set => AppSettings.AddOrUpdateValue("DefaultLogViewFlat", value); + } + #region Catalogue flag visibility settings public static bool ShowInternalCatalogues @@ -209,24 +219,12 @@ public static bool ShowDeprecatedCatalogues set => AppSettings.AddOrUpdateValue("ShowDeprecatedCatalogues", value); } - public static bool ShowColdStorageCatalogues - { - get => AppSettings.GetValueOrDefault("ShowColdStorageCatalogues", true); - set => AppSettings.AddOrUpdateValue("ShowColdStorageCatalogues", value); - } - public static bool ShowProjectSpecificCatalogues { get => AppSettings.GetValueOrDefault("ShowProjectSpecificCatalogues", true); set => AppSettings.AddOrUpdateValue("ShowProjectSpecificCatalogues", value); } - public static bool ShowNonExtractableCatalogues - { - get => AppSettings.GetValueOrDefault("ShowNonExtractableCatalogues", true); - set => AppSettings.AddOrUpdateValue("ShowNonExtractableCatalogues", value); - } - /// /// True to apply theme changes to context menus and tool strips. /// diff --git a/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs b/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs index d0463360d8..91d9d1ffeb 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/RunUITests.cs @@ -31,7 +31,6 @@ private List allowedToBeIncompatible typeof(ExecuteCommandSetDataAccessContextForCredentials), typeof(ExecuteCommandActivate), typeof(ExecuteCommandCreateNewExternalDatabaseServer), - typeof(ExecuteCommandShowTooltip), typeof(ExecuteCommandShowKeywordHelp), typeof(ExecuteCommandCollapseChildNodes), typeof(ExecuteCommandExpandAllNodes), @@ -75,7 +74,8 @@ private List allowedToBeIncompatible typeof(ExecuteCommandSetArgument), typeof(ExecuteCommandAddToSession), typeof(ExecuteCommandDeletePlugin), - typeof(ExecuteCommandPerformRegexRedactionOnCatalogue) + typeof(ExecuteCommandPerformRegexRedactionOnCatalogue), + typeof(ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider) }); [Test] diff --git a/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs b/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs index a20dc80d11..a69c6496a2 100644 --- a/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs +++ b/Rdmp.UI/CatalogueSummary/LoadEvents/LoadEventsTreeView.cs @@ -8,6 +8,7 @@ using System.Collections; using System.Collections.Generic; using System.ComponentModel; +using System.DirectoryServices.ActiveDirectory; using System.Drawing; using System.Linq; using System.Text; @@ -20,6 +21,7 @@ using Rdmp.Core.Logging; using Rdmp.Core.Logging.PastEvents; using Rdmp.Core.ReusableLibraryCode; +using Rdmp.Core.ReusableLibraryCode.Settings; using Rdmp.UI.Collections; using Rdmp.UI.ItemActivation; using Rdmp.UI.Menus.MenuItems; @@ -50,7 +52,9 @@ public partial class LoadEventsTreeView : RDMPUserControl, IObjectCollectionCont private readonly ToolStripButton _btnApplyFilter = new("Apply"); private readonly ToolStripTextBox _tbToFetch = new() { Text = "1000" }; private readonly ToolStripButton _btnFetch = new("Go"); + private readonly ToolStripButton _btnFlat = new("Flat View"); + private bool _flatView = UserSettings.DefaultLogViewFlat; private int _toFetch = 1000; @@ -58,7 +62,7 @@ public partial class LoadEventsTreeView : RDMPUserControl, IObjectCollectionCont public LoadEventsTreeView() { InitializeComponent(); - + if (_flatView) _btnFlat.Text = "Nested View"; _populateLoadHistory.DoWork += _populateLoadHistory_DoWork; _populateLoadHistory.WorkerSupportsCancellation = true; _populateLoadHistory.RunWorkerCompleted += _populateLoadHistory_RunWorkerCompleted; @@ -78,7 +82,7 @@ public LoadEventsTreeView() _btnApplyFilter.Click += (s, e) => ApplyFilter(_tbFilterBox.Text); _tbToFetch.TextChanged += TbToFetchTextChanged; _btnFetch.Click += (s, e) => PopulateLoadHistory(); - + _btnFlat.Click += (s,e) => ToggleView(); RDMPCollectionCommonFunctionality.SetupColumnTracking(treeView1, olvDescription, new Guid("6b09f39c-2b88-41ed-a396-42a2d2288952")); RDMPCollectionCommonFunctionality.SetupColumnTracking(treeView1, olvDate, @@ -101,6 +105,7 @@ private void TbToFetchTextChanged(object sender, EventArgs e) private object olvDescription_AspectGetter(object rowObject) { + if (rowObject is null) return null; return rowObject switch { ArchivalDataLoadInfo adi => adi.ToString(), @@ -117,6 +122,7 @@ private object olvDescription_AspectGetter(object rowObject) private static object olvDate_AspectGetter(object rowObject) { + if (rowObject is null) return null; return rowObject switch { ArchivalDataLoadInfo adi => adi.StartTime, @@ -128,9 +134,15 @@ private static object olvDate_AspectGetter(object rowObject) }; } - private static void treeView1_FormatRow(object sender, FormatRowEventArgs e) + private void treeView1_FormatRow(object sender, FormatRowEventArgs e) { // Only apply if it is a data load info thing + if (_flatView) + { + if (e.Model is ArchivalFatalError atli) + e.Item.ForeColor = Color.DarkOrange; + } + if (e.Model is not ArchivalDataLoadInfo dli) return; if (dli.HasErrors) @@ -206,7 +218,15 @@ private void _populateLoadHistory_RunWorkerCompleted(object sender, RunWorkerCom public void AddObjects(ArchivalDataLoadInfo[] archivalDataLoadInfos) { - treeView1.AddObjects(archivalDataLoadInfos); + if (_flatView) { + treeView1.AddObjects(archivalDataLoadInfos.SelectMany(adli => adli.TableLoadInfos).ToList()); + treeView1.AddObjects(archivalDataLoadInfos.SelectMany(adli => adli.Errors).ToList()); + treeView1.AddObjects(archivalDataLoadInfos.SelectMany(adli => adli.Progress).ToList()); + } + else + { + treeView1.AddObjects(archivalDataLoadInfos); + } } public void ClearObjects() @@ -241,6 +261,13 @@ private void _populateLoadHistory_DoWork(object sender, DoWorkEventArgs e) } } + private void ToggleView() + { + _flatView = !_flatView; + _btnFlat.Text = _flatView ?"Nested View": "Flat View"; + PopulateLoadHistory(); + } + private void PopulateLoadHistory() { //it's already doing it... @@ -282,11 +309,110 @@ private void llLoading_LinkClicked(object sender, LinkLabelLinkClickedEventArgs AbortWorkers(); } + private class LogFilter : AbstractModelFilter + { + public LogFilter() + { + + } + + public LogFilter(ObjectListView olv) + { + this.ListView = olv; + } + + public LogFilter(ObjectListView olv, string text) + { + this.ListView = olv; + this.Text = text; + } + + public LogFilter(ObjectListView olv, string text, StringComparison comparison) + { + this.ListView = olv; + this.Text = text; + this.StringComparison = comparison; + } + + public string Text; + public StringComparison StringComparison = StringComparison.InvariantCultureIgnoreCase; + + protected ObjectListView ListView; + + private List acceptableChildren = []; + private List acceptableRoots = []; + + public override bool Filter(object modelObject) + { + if (this.ListView == null || String.IsNullOrEmpty(this.Text)) + return true; + + foreach (OLVColumn column in this.ListView.Columns) + { + if (column.IsVisible) + { + string cellText = column.GetStringValue(modelObject); + if (cellText.IndexOf(this.Text, this.StringComparison) != -1) + { + if (modelObject is ArchivalDataLoadInfo adli) + { + acceptableChildren.AddRange(adli.Progress.Select(p => p.ID)); + acceptableRoots.Add(adli.ID); + } + return true; + } + else if (modelObject is ArchivalProgressLog log) + { + if (acceptableChildren.Contains(log.ID)) + { + return true; + } + } + else if (modelObject is LoadEventsTreeView_Category lec) + { + if (acceptableRoots.Contains(lec.RunId)) + { + return true; + } + if (lec.Children is ArchivalProgressLog[] lapl) + { + if (lapl.Any(pl => column.GetStringValue(pl).IndexOf(this.Text, this.StringComparison) != -1)) return true; + } + if (lec.Children is ArchivalTableLoadInfo[] latli) + { + if (latli.Any(pl => column.GetStringValue(pl).IndexOf(this.Text, this.StringComparison) != -1)) return true; + } + if (lec.Children is ArchivalFatalError[] lafe) + { + if (lafe.Any(pl => column.GetStringValue(pl).IndexOf(this.Text, this.StringComparison) != -1)) return true; + } + } + else if (modelObject is ArchivalTableLoadInfo dli) + { + if (acceptableRoots.Contains(dli.Parent.ID)) + { + return true; + } + } + else if (modelObject is ArchivalDataLoadInfo adli) + { + if (adli.Progress.Any(p => column.GetStringValue(p).IndexOf(this.Text, this.StringComparison) != -1)) return true; + if (adli.TableLoadInfos.Any(p => column.GetStringValue(p).IndexOf(this.Text, this.StringComparison) != -1)) return true; + if (adli.Errors.Any(p => column.GetStringValue(p).IndexOf(this.Text, this.StringComparison) != -1)) return true; + } + } + } + + return false; + } + } public void ApplyFilter(string filter) { - treeView1.ModelFilter = new TextMatchFilter(treeView1, filter, StringComparison.CurrentCultureIgnoreCase); + + treeView1.ModelFilter = new LogFilter(treeView1, filter, StringComparison.CurrentCultureIgnoreCase); treeView1.UseFiltering = !string.IsNullOrWhiteSpace(filter); + treeView1.DefaultRenderer = new HighlightTextRenderer(new TextMatchFilter(treeView1, filter)); } private void treeView1_ColumnRightClick(object sender, CellRightClickEventArgs e) @@ -406,7 +532,7 @@ public void SetCollection(IActivateItems activator, IPersistableObjectCollection CommonFunctionality.Add(new ToolStripLabel("Fetch:")); CommonFunctionality.Add(_tbToFetch); CommonFunctionality.Add(_btnFetch); - + CommonFunctionality.Add(_btnFlat); PopulateLoadHistory(); } } \ No newline at end of file diff --git a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs index 0abec794e8..93c5a626c6 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.Designer.cs @@ -28,101 +28,88 @@ protected override void Dispose(bool disposing) /// private void InitializeComponent() { - this.cbShowColdStorage = new System.Windows.Forms.CheckBox(); - this.cbShowDeprecated = new System.Windows.Forms.CheckBox(); - this.cbShowInternal = new System.Windows.Forms.CheckBox(); - this.cbProjectSpecific = new System.Windows.Forms.CheckBox(); - this.cbShowNonExtractable = new System.Windows.Forms.CheckBox(); - this.flowLayoutPanel1 = new System.Windows.Forms.FlowLayoutPanel(); - this.flowLayoutPanel1.SuspendLayout(); - this.SuspendLayout(); - // - // cbShowColdStorage - // - this.cbShowColdStorage.AutoSize = true; - this.cbShowColdStorage.Location = new System.Drawing.Point(3, 3); - this.cbShowColdStorage.Name = "cbShowColdStorage"; - this.cbShowColdStorage.Size = new System.Drawing.Size(87, 17); - this.cbShowColdStorage.TabIndex = 5; - this.cbShowColdStorage.Text = "Cold Storage"; - this.cbShowColdStorage.UseVisualStyleBackColor = true; - this.cbShowColdStorage.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); + cbShowDeprecated = new System.Windows.Forms.CheckBox(); + cbShowInternal = new System.Windows.Forms.CheckBox(); + cbProjectSpecific = new System.Windows.Forms.CheckBox(); + flowLayoutPanel1 = new System.Windows.Forms.FlowLayoutPanel(); + cbFlatView = new System.Windows.Forms.CheckBox(); + flowLayoutPanel1.SuspendLayout(); + SuspendLayout(); // // cbShowDeprecated // - this.cbShowDeprecated.AutoSize = true; - this.cbShowDeprecated.Location = new System.Drawing.Point(163, 3); - this.cbShowDeprecated.Name = "cbShowDeprecated"; - this.cbShowDeprecated.Size = new System.Drawing.Size(82, 17); - this.cbShowDeprecated.TabIndex = 6; - this.cbShowDeprecated.Text = "Deprecated"; - this.cbShowDeprecated.UseVisualStyleBackColor = true; - this.cbShowDeprecated.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); + cbShowDeprecated.AutoSize = true; + cbShowDeprecated.Location = new System.Drawing.Point(75, 3); + cbShowDeprecated.Name = "cbShowDeprecated"; + cbShowDeprecated.Size = new System.Drawing.Size(86, 19); + cbShowDeprecated.TabIndex = 6; + cbShowDeprecated.Text = "Deprecated"; + cbShowDeprecated.UseVisualStyleBackColor = true; + cbShowDeprecated.CheckedChanged += OnCheckboxChanged; // // cbShowInternal // - this.cbShowInternal.AutoSize = true; - this.cbShowInternal.Location = new System.Drawing.Point(96, 3); - this.cbShowInternal.Name = "cbShowInternal"; - this.cbShowInternal.Size = new System.Drawing.Size(61, 17); - this.cbShowInternal.TabIndex = 7; - this.cbShowInternal.Text = "Internal"; - this.cbShowInternal.UseVisualStyleBackColor = true; - this.cbShowInternal.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); + cbShowInternal.AutoSize = true; + cbShowInternal.Location = new System.Drawing.Point(3, 3); + cbShowInternal.Name = "cbShowInternal"; + cbShowInternal.Size = new System.Drawing.Size(66, 19); + cbShowInternal.TabIndex = 7; + cbShowInternal.Text = "Internal"; + cbShowInternal.UseVisualStyleBackColor = true; + cbShowInternal.CheckedChanged += OnCheckboxChanged; // // cbProjectSpecific // - this.cbProjectSpecific.AutoSize = true; - this.cbProjectSpecific.Location = new System.Drawing.Point(3, 26); - this.cbProjectSpecific.Name = "cbProjectSpecific"; - this.cbProjectSpecific.Size = new System.Drawing.Size(100, 17); - this.cbProjectSpecific.TabIndex = 8; - this.cbProjectSpecific.Text = "Project Specific"; - this.cbProjectSpecific.UseVisualStyleBackColor = true; - this.cbProjectSpecific.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); + cbProjectSpecific.AutoSize = true; + cbProjectSpecific.Location = new System.Drawing.Point(3, 28); + cbProjectSpecific.Name = "cbProjectSpecific"; + cbProjectSpecific.Size = new System.Drawing.Size(107, 19); + cbProjectSpecific.TabIndex = 8; + cbProjectSpecific.Text = "Project Specific"; + cbProjectSpecific.UseVisualStyleBackColor = true; + cbProjectSpecific.CheckedChanged += OnCheckboxChanged; // - // cbShowNonExtractable + // flowLayoutPanel1 // - this.cbShowNonExtractable.AutoSize = true; - this.cbShowNonExtractable.Location = new System.Drawing.Point(109, 26); - this.cbShowNonExtractable.Name = "cbShowNonExtractable"; - this.cbShowNonExtractable.Size = new System.Drawing.Size(102, 17); - this.cbShowNonExtractable.TabIndex = 9; - this.cbShowNonExtractable.Text = "Non Extractable"; - this.cbShowNonExtractable.UseVisualStyleBackColor = true; - this.cbShowNonExtractable.CheckedChanged += new System.EventHandler(this.OnCheckboxChanged); + flowLayoutPanel1.Controls.Add(cbShowInternal); + flowLayoutPanel1.Controls.Add(cbShowDeprecated); + flowLayoutPanel1.Controls.Add(cbProjectSpecific); + flowLayoutPanel1.Controls.Add(cbFlatView); + flowLayoutPanel1.Dock = System.Windows.Forms.DockStyle.Fill; + flowLayoutPanel1.Location = new System.Drawing.Point(0, 0); + flowLayoutPanel1.Name = "flowLayoutPanel1"; + flowLayoutPanel1.Size = new System.Drawing.Size(256, 69); + flowLayoutPanel1.TabIndex = 10; // - // flowLayoutPanel1 + // cbFlatView // - this.flowLayoutPanel1.Controls.Add(this.cbShowColdStorage); - this.flowLayoutPanel1.Controls.Add(this.cbShowInternal); - this.flowLayoutPanel1.Controls.Add(this.cbShowDeprecated); - this.flowLayoutPanel1.Controls.Add(this.cbProjectSpecific); - this.flowLayoutPanel1.Controls.Add(this.cbShowNonExtractable); - this.flowLayoutPanel1.Dock = System.Windows.Forms.DockStyle.Fill; - this.flowLayoutPanel1.Location = new System.Drawing.Point(0, 0); - this.flowLayoutPanel1.Name = "flowLayoutPanel1"; - this.flowLayoutPanel1.Size = new System.Drawing.Size(256, 69); - this.flowLayoutPanel1.TabIndex = 10; + cbFlatView.Appearance = System.Windows.Forms.Appearance.Button; + cbFlatView.AutoSize = true; + cbFlatView.Location = new System.Drawing.Point(116, 28); + cbFlatView.Name = "cbFlatView"; + cbFlatView.Size = new System.Drawing.Size(64, 25); + cbFlatView.TabIndex = 9; + cbFlatView.Text = "Flat View"; + cbFlatView.UseVisualStyleBackColor = true; + cbFlatView.CheckedChanged += OnCheckboxChanged; // // CatalogueCollectionFilterUI // - this.Controls.Add(this.flowLayoutPanel1); - this.Name = "CatalogueCollectionFilterUI"; - this.Size = new System.Drawing.Size(256, 69); - this.flowLayoutPanel1.ResumeLayout(false); - this.flowLayoutPanel1.PerformLayout(); - this.ResumeLayout(false); + Controls.Add(flowLayoutPanel1); + Name = "CatalogueCollectionFilterUI"; + Size = new System.Drawing.Size(256, 69); + flowLayoutPanel1.ResumeLayout(false); + flowLayoutPanel1.PerformLayout(); + ResumeLayout(false); } #endregion - private System.Windows.Forms.CheckBox cbShowColdStorage; private System.Windows.Forms.CheckBox cbShowDeprecated; private System.Windows.Forms.CheckBox cbShowInternal; private System.Windows.Forms.CheckBox cbProjectSpecific; - private System.Windows.Forms.CheckBox cbShowNonExtractable; private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel1; + private System.Windows.Forms.CheckBox cbFlatView; } } diff --git a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs index 5464cf7c63..4c1f653f92 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.cs @@ -21,9 +21,8 @@ public CatalogueCollectionFilterUI() cbShowInternal.Checked = UserSettings.ShowInternalCatalogues; cbShowDeprecated.Checked = UserSettings.ShowDeprecatedCatalogues; - cbShowColdStorage.Checked = UserSettings.ShowColdStorageCatalogues; cbProjectSpecific.Checked = UserSettings.ShowProjectSpecificCatalogues; - cbShowNonExtractable.Checked = UserSettings.ShowNonExtractableCatalogues; + cbFlatView.Checked = UserSettings.ShowFlatLists; _loading = false; } @@ -37,27 +36,22 @@ private void OnCheckboxChanged(object sender, EventArgs e) UserSettings.ShowInternalCatalogues = cbShowInternal.Checked; UserSettings.ShowDeprecatedCatalogues = cbShowDeprecated.Checked; - UserSettings.ShowColdStorageCatalogues = cbShowColdStorage.Checked; UserSettings.ShowProjectSpecificCatalogues = cbProjectSpecific.Checked; - UserSettings.ShowNonExtractableCatalogues = cbShowNonExtractable.Checked; - + UserSettings.ShowFlatLists = cbFlatView.Checked; FiltersChanged?.Invoke(this, EventArgs.Empty); } public void EnsureVisible(Catalogue c) { - if (c.IsColdStorageDataset || c.IsDeprecated || c.IsInternalDataset) + if ( c.IsDeprecated || c.IsInternalDataset) { //trouble is our flags might be hiding it so make sure it is visible - cbShowColdStorage.Checked = cbShowColdStorage.Checked || c.IsColdStorageDataset; cbShowDeprecated.Checked = cbShowDeprecated.Checked || c.IsDeprecated; cbShowInternal.Checked = cbShowInternal.Checked || c.IsInternalDataset; } var isExtractable = c.GetExtractabilityStatus(null); - cbShowNonExtractable.Checked = cbShowNonExtractable.Checked || isExtractable == null || - isExtractable.IsExtractable == false; } /// @@ -72,13 +66,7 @@ public void CheckForChanges() if (cbShowDeprecated.Checked != UserSettings.ShowDeprecatedCatalogues) cbShowDeprecated.Checked = UserSettings.ShowDeprecatedCatalogues; - if (cbShowColdStorage.Checked != UserSettings.ShowColdStorageCatalogues) - cbShowColdStorage.Checked = UserSettings.ShowColdStorageCatalogues; - if (cbProjectSpecific.Checked != UserSettings.ShowProjectSpecificCatalogues) cbProjectSpecific.Checked = UserSettings.ShowProjectSpecificCatalogues; - - if (cbShowNonExtractable.Checked != UserSettings.ShowNonExtractableCatalogues) - cbShowNonExtractable.Checked = UserSettings.ShowNonExtractableCatalogues; } } \ No newline at end of file diff --git a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.resx b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.resx index 1af7de150c..8b2ff64a11 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionFilterUI.resx +++ b/Rdmp.UI/Collections/CatalogueCollectionFilterUI.resx @@ -1,17 +1,17 @@  - diff --git a/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs b/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs index f5393fc83a..bb5ee10b07 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionUI.Designer.cs @@ -29,12 +29,12 @@ private void InitializeComponent() olvFilters = new OLVColumn(); olvOrder = new OLVColumn(); imageList_RightClickIcons = new ImageList(components); - gbColdStorage = new GroupBox(); + gbCatalogueFilters = new GroupBox(); + tbFilter = new TextBox(); catalogueCollectionFilterUI1 = new CatalogueCollectionFilterUI(); panel2 = new Panel(); - tbFilter = new TextBox(); ((ISupportInitialize)tlvCatalogues).BeginInit(); - gbColdStorage.SuspendLayout(); + gbCatalogueFilters.SuspendLayout(); panel2.SuspendLayout(); SuspendLayout(); // @@ -49,7 +49,7 @@ private void InitializeComponent() tlvCatalogues.Location = new System.Drawing.Point(0, 0); tlvCatalogues.Name = "tlvCatalogues"; tlvCatalogues.ShowGroups = false; - tlvCatalogues.Size = new System.Drawing.Size(500, 414); + tlvCatalogues.Size = new System.Drawing.Size(500, 403); tlvCatalogues.TabIndex = 0; tlvCatalogues.Text = "label1"; tlvCatalogues.UseCompatibleStateImageBehavior = false; @@ -90,51 +90,51 @@ private void InitializeComponent() imageList_RightClickIcons.Images.SetKeyName(3, "LOG"); imageList_RightClickIcons.Images.SetKeyName(4, "aggregates.png"); // - // gbColdStorage + // gbCatalogueFilters // - gbColdStorage.Controls.Add(tbFilter); - gbColdStorage.Controls.Add(catalogueCollectionFilterUI1); - gbColdStorage.Dock = DockStyle.Bottom; - gbColdStorage.Location = new System.Drawing.Point(0, 414); - gbColdStorage.Name = "gbColdStorage"; - gbColdStorage.Size = new System.Drawing.Size(500, 65); - gbColdStorage.TabIndex = 1; - gbColdStorage.TabStop = false; - gbColdStorage.Text = "Show"; + gbCatalogueFilters.Controls.Add(tbFilter); + gbCatalogueFilters.Controls.Add(catalogueCollectionFilterUI1); + gbCatalogueFilters.Dock = DockStyle.Bottom; + gbCatalogueFilters.Location = new System.Drawing.Point(0, 403); + gbCatalogueFilters.Name = "gbCatalogueFilters"; + gbCatalogueFilters.Size = new System.Drawing.Size(500, 76); + gbCatalogueFilters.TabIndex = 1; + gbCatalogueFilters.TabStop = false; + gbCatalogueFilters.Text = "Show"; + // + // tbFilter + // + tbFilter.Location = new System.Drawing.Point(0, 53); + tbFilter.Name = "tbFilter"; + tbFilter.Size = new System.Drawing.Size(500, 23); + tbFilter.TabIndex = 2; // // catalogueCollectionFilterUI1 // catalogueCollectionFilterUI1.Dock = DockStyle.Fill; catalogueCollectionFilterUI1.Location = new System.Drawing.Point(3, 19); catalogueCollectionFilterUI1.Name = "catalogueCollectionFilterUI1"; - catalogueCollectionFilterUI1.Size = new System.Drawing.Size(494, 43); + catalogueCollectionFilterUI1.Size = new System.Drawing.Size(494, 54); catalogueCollectionFilterUI1.TabIndex = 0; // // panel2 // panel2.Controls.Add(tlvCatalogues); - panel2.Controls.Add(gbColdStorage); + panel2.Controls.Add(gbCatalogueFilters); panel2.Dock = DockStyle.Fill; panel2.Location = new System.Drawing.Point(0, 0); panel2.Name = "panel2"; panel2.Size = new System.Drawing.Size(500, 479); panel2.TabIndex = 2; // - // tbFilter - // - tbFilter.Location = new System.Drawing.Point(0, 39); - tbFilter.Name = "tbFilter"; - tbFilter.Size = new System.Drawing.Size(500, 23); - tbFilter.TabIndex = 2; - // // CatalogueCollectionUI // Controls.Add(panel2); Name = "CatalogueCollectionUI"; Size = new System.Drawing.Size(500, 479); ((ISupportInitialize)tlvCatalogues).EndInit(); - gbColdStorage.ResumeLayout(false); - gbColdStorage.PerformLayout(); + gbCatalogueFilters.ResumeLayout(false); + gbCatalogueFilters.PerformLayout(); panel2.ResumeLayout(false); ResumeLayout(false); } @@ -145,10 +145,10 @@ private void InitializeComponent() private ImageList imageList_RightClickIcons; private OLVColumn olvColumn1; private OLVColumn olvFilters; - private GroupBox gbColdStorage; + private GroupBox gbCatalogueFilters; private OLVColumn olvOrder; private CatalogueCollectionFilterUI catalogueCollectionFilterUI1; private Panel panel2; private TextBox tbFilter; } -} +} \ No newline at end of file diff --git a/Rdmp.UI/Collections/CatalogueCollectionUI.cs b/Rdmp.UI/Collections/CatalogueCollectionUI.cs index 7a1c34deb4..78f53fa55c 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionUI.cs +++ b/Rdmp.UI/Collections/CatalogueCollectionUI.cs @@ -16,6 +16,7 @@ using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Icons.IconProvision; using Rdmp.Core.Providers.Nodes; +using Rdmp.Core.ReusableLibraryCode.Settings; using Rdmp.UI.Collections.Providers.Filtering; using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.ItemActivation; @@ -49,6 +50,8 @@ public partial class CatalogueCollectionUI : RDMPCollectionUI private bool bLoading = true; + private bool _renderingFlatFiew = false; + public CatalogueCollectionUI() { InitializeComponent(); @@ -107,8 +110,19 @@ public void RefreshUIFromDatabase(object oRefreshFrom) var newCatalogues = CommonTreeFunctionality.CoreChildProvider.AllCatalogues.Except(_allCatalogues); if (newCatalogues.Any()) { - oRefreshFrom = rootFolder; //refresh from the root instead - tlvCatalogues.RefreshObject(oRefreshFrom); + if (UserSettings.ShowFlatLists) + { + oRefreshFrom = null; + tlvCatalogues.AddObjects(newCatalogues.ToList()); + tlvCatalogues.RefreshObjects(Activator.CoreChildProvider.AllCatalogues); + _renderingFlatFiew = true; + } + else + { + oRefreshFrom = rootFolder; //refresh from the root instead + tlvCatalogues.RefreshObject(oRefreshFrom); + _renderingFlatFiew = false; + } } } @@ -184,6 +198,21 @@ public void ApplyFilters() tlvCatalogues.UseFiltering = true; tlvCatalogues.ModelFilter = new CatalogueCollectionFilter(Activator.CoreChildProvider); + if (UserSettings.ShowFlatLists && !_renderingFlatFiew) + { + //reset to flat view + tlvCatalogues.RemoveObject(Activator.CoreChildProvider.CatalogueRootFolder); + tlvCatalogues.AddObjects(Activator.CoreChildProvider.AllCatalogues); + _renderingFlatFiew = true; + } + else if(!UserSettings.ShowFlatLists && _renderingFlatFiew) + { + //reset to folder view + tlvCatalogues.RemoveObjects(Activator.CoreChildProvider.AllCatalogues); + tlvCatalogues.AddObject(Activator.CoreChildProvider.CatalogueRootFolder); + tlvCatalogues.RefreshObject(Activator.CoreChildProvider.CatalogueRootFolder); + _renderingFlatFiew = false; + } } public enum HighlightCatalogueType @@ -259,7 +288,16 @@ public override void SetItemActivator(IActivateItems activator) Activator.RefreshBus.EstablishLifetimeSubscription(this); tlvCatalogues.AddObject(activator.CoreChildProvider.AllGovernanceNode); - tlvCatalogues.AddObject(activator.CoreChildProvider.CatalogueRootFolder); + if (UserSettings.ShowFlatLists) + { + tlvCatalogues.AddObjects(activator.CoreChildProvider.AllCatalogues); + _renderingFlatFiew = true; + } + else + { + tlvCatalogues.AddObject(activator.CoreChildProvider.CatalogueRootFolder); + _renderingFlatFiew = false; + } ApplyFilters(); RefreshUIFromDatabase(activator.CoreChildProvider.CatalogueRootFolder); diff --git a/Rdmp.UI/Collections/CatalogueCollectionUI.resx b/Rdmp.UI/Collections/CatalogueCollectionUI.resx index 4e69981475..1dc9e525a6 100644 --- a/Rdmp.UI/Collections/CatalogueCollectionUI.resx +++ b/Rdmp.UI/Collections/CatalogueCollectionUI.resx @@ -1,7 +1,7 @@  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + + AAABAAUAQEAAAAEAIAAoQgAAVgAAACAgAAABACAAqBAAAH5CAAAgIAAAAQAIAKgIAAAmUwAAEBAAAAEA + IABoBAAAzlsAABAQAAABAAgAaAUAADZgAAAoAAAAQAAAAIAAAAABACAAAAAAAACAAAAAAAAAAAAAAAAA + AAAAAAAA//////39/f/29vb/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/29vb//f39///////39/f/xcXF/7Kysv+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gx + sf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gx + sf+xsbH/sbGx/7CwsP+wsLD/sLCw/7CwsP+wsLD/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gx + sf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gxsf+xsbH/sbGx/7Gx + sf+xsbH/sbGx/7CwsP+wsLD/sLCw/7CwsP+xsbH/xMTE//f39///////9fX1/9vb2//39/f/9vb2//b2 + 9v/29vb/+fn5//r6+v/6+vr/+vr6//f39//39/f/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+fn5//b2 + 9v/39/f/9/f3//f39//39/f/9/f3//f39//4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/39/f/9/f3//j4 + +P/4+Pj/+fn5//n5+f/6+vr/+vr6//f39//39/f/9/f3//f39//4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4 + +P/39/f/9/f3//f39//39/f/9/f3//f39//39/f/9/f3//f39//39/f/9/f3/9jY2P/09PT///////X1 + 9f/b29v///////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + ///a2tr/8/Pz///////19fX/29vb////////////9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T0 + 9P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T0 + 9P/09PT/9PT0//T09P/09PT/8/Pz//T09P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T0 + 9P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T09P/09PT/9PT0//T0 + 9P/09PT/9PT0//7+/v//////2tra//Pz8///////9fX1/9vb2////////////+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//ycnk/9XV6P/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9ra2v/z8/P///////X19f/c3Nz///////// + ///v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+vp4f+/p0b/spMX/7GTFv++pkP/6+ni/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/8fH4//T0+j/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////a2tr/8/Pz//// + ///19fX/2tra////////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/9bJl/+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv/Ux5D/7+/v/+/v7//v7+//7+/v/+/v7//FxeP/0dHm/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+ + /v//////2NjY//Pz8///////9fX1/9XV1f///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+Pd + xP+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/+bi0f/v7+//7+/v/+/v7//v7+//w8Pi/9DQ + 5v/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//+/v7//////9jY2P/z8/P///////X19f/V1dX////////////v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7/+4nC3/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+wkRL/7+/v/+/v + 7//v7+//7+/v/8DA4v/OzuX/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////Y2Nj/8/Pz///////19fX/1dXV//// + ////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/+7t6//v7+//7+/v/+/v7/+/v+H/zc3l/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2NjY//Pz + 8///////9fX1/9XV1f///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv/t7Oj/7+/v/+/v7//v7+//vLzh/8zM5f/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//+/v7//////9nZ2f/z8/P///////X19f/S0tL////////////v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+DY + uv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/493F/+/v7//v7+//7+/v/7q6 + 4P/KyuT/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+///v7+///////X19f/8/Pz///////19fX/0tLS////////////7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//6eba/7mdL//ZzqH/zr17/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/zLt3/9PG + j/+7oDX/6+nh/+/v7/+4uN//ycnk/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////19fX//Pz8///////9fX1/9HR + 0f///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+7/w61V/8i1af/u7ev/7+/v/+7u7f+zlRr/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/tZch/+7u7v/v7+//7e3p/8SvWf/HtGP/tbXd/8bG4v/v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9bW + 1v/z8/P///////X19f/R0dH////////////v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+7u7f/Ux5P/u6E4/+rn3v/v7+//7+/v/+/v7//v7+//7+/u/8Ot + Vf+tjAj/rYwG/62MBv+tjQj/wqtR/+7u7f/v7+//7+/v/+/v7//v7+//6OXZ/7GXMf/Gu53/7u7u/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+///v7+///////W1tb/8/Pz///////19fX/0dHR////////////7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//4tvA/7idL/+wkBH/tJYd/9fMnf/v7+7/7+/v/+PeyP+ykxn/5N/K/+7u7v/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7u3s/+vp4f/r6eD/7u3r/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7/+wsN3/ubK1/7KUG//o5Nb/7+/v/+7u7v/YzZ//tJYf/7CQEf+3myv/4Nq+/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////19fX//Pz8///////9fX1/8/Pz////////////+/v + 7//v7+//7+/v/+/v7//u7u7/u6I5/62MBv+tjAb/rYwG/62MBv+tjAb/spQa/7ugNv/Ux5H/7u7u/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//ra3c/8PD4v/u7u3/0MGD/76mRf+xkxb/rYwG/62MBv+tjAb/rYwG/62M + Bv+6oDb/7u7t/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9fX1//z8/P///////X1 + 9f/Pz8/////////////v7+//7+/v/+/v7//v7+//wqtQ/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+wkRL/7+/u/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/6mp2//BweH/7+/v/+7u7v+0lh//rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/8GqTf/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+//// + ///X19f/8/Pz///////19fX/z8/P////////////7+/v/+/v7//v7+//6+nh/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/93Vsv/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+np9r/v7/h/+/v + 7//h2rz/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/7Ovm/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v//7+/v//////19fX//Pz8///////9fX1/8/Pz////////////+/v7//v7+//7+/v/9rQ + pv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv/DrVP/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//pKTZ/76+4f/v7+//vqZD/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/9rP + pv/v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9fX1//z8/P///////X19f/Ozs7///////// + ///v7+//7+/v/+/v7//WyZX/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/vaRA/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/6Cg2P+8vOD/7+/v/7yiOv+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv/VyJP/7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////X19f/8/Pz//// + ///19fX/0NDQ////////////7+/v/+/v7//v7+//493F/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/829ev/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+dndj/w8Pi/+/v7//Pv37/rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/5eDL/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+ + /v//////2dnZ//Pz8///////9fX1/8/Pz////////////+/v7//v7+//7+/v/+7u7v+vjw//rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+Cf8D/7Ozu/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//lJTV/7q6 + 4P/v7+//7evn/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/r48O/+/v7v/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//+/v7//////9nZ2f/z8/P///////X19f/Ozs7////////////v7+//7+/v/+/v + 7//v7+//5N3H/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv/RwYT/29vp/yEhtP+vr93/7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/5eX1v+4uN//7+/v/+/v7//QwYP/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/+Pc + w//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////a2tr/8/Pz///////19fX/zs7O//// + ////////7+/v/+/v7//v7+//7+/v/+/v7//j3cX/s5Qa/62MBv+tjAb/rYwG/6+PDf/WyZf/7+/u/+/v + 7//v7+//o6PZ/ysruP/KyuT/7+/v/+/v7//v7+//7+/v/+7t6//t7er/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+UlNX/t7ff/+/v7//v7+//7+/v/9jMnP+vjw7/rYwG/62M + Bv+tjAb/spQa/+Lbwf/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2tra//Pz + 8///////9fX1/8/Pz////////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//t7Or/s5Qb/+zq + 5P/u7u7/7+/v/+/v7//v7+//7+/v/+/v7//u7u7/ZWXH/z4+vf/p6e3/7u7s/7icLv+tjAb/rYwG/7GT + F//q6N7/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//kZHU/7W13//v7+//7+/v/+/v + 7//v7+//7u7u/+vq5P+7oDf/7ezp/+7u7v/v7+//7+/v/+/v7//v7+//7+/v/+/v7//q6u7/Tk7B/+/v + 7//+/v7//////9ra2v/z8/P///////X19f/S0tL////////////v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/7SWH//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//g4Or/MjK4/6qL + FP+tjAb/rYwG/62MBv+tjAb/r48N/+7u7v/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/46O + 1P+zs97/7+/v/+/v7//v7+//7+/v/+/v7//u7u7/vaM+/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+3t + 7v9+fs7/LCy3/9nZ6f/v7+///v7+///////Z2dn/8/Pz///////19fX/0tLS////////////7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+0lh//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+Lbw/+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv/VyZX/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7/+Li9L/srLd/+/v7//v7+//7+/v/+/v7//v7+//7u7u/72jPv/v7+//7+/v/+/v + 7//v7+//7+/v/7Cw3f8REa//urrg/+7u7v/v7+//7+/v//7+/v//////2NjY//Pz8///////9fX1/9LS + 0v///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//tJYf/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//ZzaH/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/w65W/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//h4fR/7Gx3f/v7+//7+/v/+/v7//v7+//7+/v/+7u + 7v+9oz7/7+/v/+/v7//v7+//zMzk/ycntv94eM3/7u7v/+/v7//v7+//7+/v/+/v7//+/v7//////9jY + 2P/z8/P///////T09P/S0tL////////////v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/7SW + H//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//5eHO/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/9fNpv/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/4SE0P+wsN3/7+/v/+/v + 7//v7+//7+/v/+/v7//u7u7/vaM+/+/v7//q6u3/Ojq7/zQ0uv/t7e//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+///v7+///////X19f/8/Pz///////19fX/0tLS////////////7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7/+0lh//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7v+3myz/rYwG/62MBv+tjAb/rYwG/7CSGf8VFbH/dHTM/+3t7v/v7+//7+/v/+/v7//v7+//7+/v/+/v + 7/+AgM//rq7c/+/v7//v7+//7+/v/+/v7//v7+//7u7u/7mfPf9kZMj/ICC0/8bG4//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2NjY//Pz8///////9fX1/9LS0v///////////+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//tJYf/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/8i1Zv+tjAf/rYwG/7+mRv/u7ev/7+/v/5mZ1/8JCa//ra3c/+7u + 7//v7+//7+/v/+/v7//v7+//fHzO/62t3P/v7+//7+/v/+/v7//v7+//7u7u/6Wl2f+HbS7/nJzX/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9nZ2f/z8/P///////X1 + 9f/T09P////////////v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/7SWH//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//6uru/01Nwf8gILT/y8vl/+/v7//v7+//7+/v/3h4zf+rq9z/7+/v/+/v7//v7+//xsbj/xwc + s/9aWsT/vaM+/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+//// + ///Z2dn/8/Pz///////19fX/09PT////////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7/+0lh//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//5eXs/xQUsf82Nrr/7Ozu/+/v7/90dMz/qqrc/+/v + 7//q6u7/Kyu3/xkZsv/k5Oz/7u7u/72jPv/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v//7+/v//////2dnZ//Pz8///////9fX1/9LS0v///////////+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//tJYf/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//u7u//n5/Y/w0N + r/9zc8v/bm7K/6Wl2v9hYcf/ERGw/7Gx3f/v7+//7+/v/+7u7v+9oz7/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9nZ2f/z8/P///////X19f/S0tL///////// + ///v7+//7+/v/+/v7//v7+//7+/v/+7u7v/n5NX/xK9a/66OCv+/qEn/4dq9/+7u7v/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//t7e//WFjE/wYGrf8GBq3/eXnN/+/v7//v7+//7u7u/76+4f9JScD/JiGa/1hY + xP/T0+f/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////Z2dn/8/Pz//// + ///19fX/0NDQ////////////7+/v/+/v7//v7+//7+/v/+/v7v/Cq1H/rYwG/62MBv+tjAb/rYwG/62M + Bv+2mSb/7e3q/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//u7u//yMjk/xYWsf8TE7D/Ghqy/ykptv/a2un/6Ojt/yIi + tP8GBq3/Bgat/wYGrf8GBq3/Bgat/05Owf/u7u7/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+ + /v//////1tbW//Pz8///////9fX1/8/Pz////////////+/v7//v7+//7+/v/+/v7//Hs2P/rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/7KTGP/u7u7/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//p6e3/NTW5/xoasv/b2+n/b2/L/6qq + 3P/IyOT/Dg6v/w8PsP8GBq3/Bgat/wYGrf8GBq3/Bgat/wYGrf8GBq3/X1/G/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//+/v7//////9fX1//z8/P///////X19f/Pz8/////////////v7+//7+/v/+/v + 7//s6uT/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/4Ni7/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+zs7v93d83/FBSx/6ys + 3P/v7+//7+/v/3Fxy/+trdz/7+/v/8XF4/8GBq3/Bgat/wYGrf8GBq3/Bgat/wYGrf8GBq3/Bgat/wYG + rf/o6O3/7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////X19f/8/Pz///////19fX/z8/P//// + ////////7+/v/+/v7//v7+//3NKr/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/8Sv + Wf/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7u7u/7q6 + 3/8MDK7/fX3O/+/v7//v7+//7+/v/+/v7/9zc8v/sLDd/+/v7/9ZWcT/Bgat/wYGrf8GBq3/Bgat/wYG + rf8GBq3/Bgat/wYGrf8GBq3/rKzc/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////19fX//Pz + 8///////9fX1/87Ozv///////////+/v7//v7+//7+/v/9XHlP+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv+9pD7/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//3d3p/yYmtv9SUsL/5+fs/+/v7//v7+//7+/v/+/v7//v7+//dXXM/7Gx3f/v7+//OTm7/wYG + rf8GBq3/Bgat/wYGrf8GBq3/Bgat/wYGrf8GBq3/Bgat/5GR1P/v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//+/v7//////9jY2P/z8/P///////X19f/Ozs7////////////v7+//7+/v/+/v7//i2r//rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/wahL/+7u7f/v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//s7O7/V1fD/yQktf/e3ur/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/3d3 + zP+0tN//7u7t/2ZfmP8GBq3/Bgat/wYGrf8GBq3/Bgat/wYGrf8GBq3/Bgat/wYGrf/ExOL/7+/v/+/v + 7//v7+//7+/v/+/v7//v7+///v7+///////Y2Nj/8/Pz///////19fX/zs7O////////////7+/v/+/v + 7//v7+//7u7s/6+ODP+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/97Wtf+3nC7/3NKr/+/u + 7v/v7+//7+/v/+/v7//v7+//7+/v/+/v7/+Tk9X/Ghqy/7e33//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7/95ec3/qJ2S/7qfNf/a1tP/Bgat/wYGrf8GBq3/Bgat/wYGrf8GBq3/Bgat/wYG + rf8GBq3/vb3g/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2dnZ//Pz8///////9fX1/8/P + z////////////+/v7//v7+//7+/v/+/v7//d1LD/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/8az + Yv/v7+//7+/v/9vRqP+0lyD/6eba/+/u7v/v7+//7+/v/87O5v8REbD/j4/U/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7v/n5Nb/rI4d/8O8sv/v7+//7+/v/2pqyf8GBq3/Bgat/wYG + rf8GBq3/Bgat/wYGrf8GBq3/rKzc/1NTwv82Nrr/5eXs/+/v7//v7+//7+/v/+/v7//+/v7//////9jY + 2P/z8/P///////X19f/Pz8/////////////v7+//7+/v/+/v7//v7+//7+/v/97Wtf+ujQr/rYwG/62M + Bv+tjAb/rYwG/9DAgf/u7u7/7+/v/+/v7//v7+//7u3s/8y6c//BqUz/5OTo/zo6vP9lZcf/6ent/+7t + 7P/Swof/vKQ+/72kQP/SxIz/7+7t/+/v7//v7+//7+/v/+3s6P+9pEH/0cGG/3t7zP+8vOD/7+/v/+/v + 7//u7u//hobR/wYGrf8GBq3/Bgat/wYGrf8KCq7/srLe/+/v7//v7+//2trp/4qK0v/v7+//7+/v/+/v + 7//v7+///v7+///////Y2Nj/8/Pz///////19fX/z8/P////////////7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+XgzP/b0Kj/4tvB/+7u7v/v7+//7+/v/+/v7//v7+//7+/v/+/v7//t7e7/dXTD/56E + Mv/Huoj/7u7t/+DZvP+tjAf/rYwG/62MBv+tjAb/rYwG/62MCP/j3sf/7u7t/828eP++pkX/7ezp/+/v + 7/9+fs//v7/h/+/v7//v7+//7+/v/+/v7//u7u//wsLi/6io2//Ly+T/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2NjY//Pz8///////9fX1/9HR0f///////////+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7/+vr93/KSm3/8bG4//v7+//5N3G/7eaKf+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rowJ/7WZ + Jf/l4c7/7+/v/+/v7//v7+//gIDP/8LC4v/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9nZ2f/z8/P///////X1 + 9f/R0dH////////////v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+Dg6/8kJLX/pKTZ/+/v7//v7+//7+/v/+/v7/+4nS//rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv+7nzX/7+/v/+/v7//v7+//7+/v/4KC0P/ExOL/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+//// + ///Z2dn/8/Pz///////19fX/0dHR////////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7e3u/1dXxP92ds3/6urt/+/v7//v7+//7+/v/+/v7//v7+//rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/+/v7//v7+//7+/v/+/v7/+EhNH/yMjj/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v//7+/v//////2dnZ//Pz8///////9fX1/9HR0f///////////+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//u7u7/l5fW/0FBvv/m5uz/7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62MBv/u7ez/7+/v/+/v + 7//v7+//hobR/8rK5f/v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7//////9ra2v/z8/P///////X19f/Q0ND///////// + ///v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//Hx+P/MzO6/9HR5v/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+tjQn/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/rYwG/62M + Bv+tjAf/7+/v/+/v7//v7+//7+/v/4iI0f/NzeX/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////a2tr/8/Pz//// + ///19fX/1dXV////////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+3t7v89Pbz/srLe/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//wqtQ/62MBv+tjAb/rYwG/62M + Bv+tjAb/rYwG/62MBv+tjAb/yLVo/+/v7//v7+//7+/v/+/v7/+KitL/0NDl/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+ + /v//////2tra//Pz8///////9fX1/9XV1f///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7e3u/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+zq + 5f+ujQr/rYwG/62MBv+tjAb/rYwG/62MBv+tjAb/sZIW/+/u7v/v7+//7+/v/+/v7//v7+//i4vS/9PT + 5//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//+/v7//////9ra2v/z8/P///////X19f/V1dX////////////v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7Ovn/7WXIv+tjAb/rYwG/62MBv+tjAb/tpop/+3s6P/v7+//7+/v/+/v + 7//v7+//7+/v/42N0//W1uj/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+///////a2tr/8/Pz///////19fX/1dXV//// + ////////7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7u7/6OXY/9rQpv/b0an/6efc/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+Pj9T/2dno/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2tra//Pz + 8///////9fX1/9XV1f///////////+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//kJDU/9zc6v/v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//+/v7//////9ra2v/z8/P///////X19f/b29v////////////v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/5SU + 1f/h4ev/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+///v7+///////a2tr/8/Pz///////19fX/29vb////////////7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v//////2dnZ//Pz8///////9fX1/9vb + 2/////////////X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X1 + 9f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X1 + 9f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X1 + 9f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/19fX/9fX1//X19f/+/v7//////9nZ + 2f/z8/P///////X19f/b29v///////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + ///////////////////Y2Nj/9PT0///////4+Pj/29vb//////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////2NjY//f39////////v7+//T09P/x8fH/8fHx//Hx + 8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx + 8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx + 8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx + 8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Pz8//+/v7///////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + /////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAACAAAABAAAAAAQAgAAAA + AAAAIAAAAAAAAAAAAAAAAAAAAAAAAPz8/P/X19f/0dHR/9HR0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR + 0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR + 0f/R0dH/0dHR/9HR0f/R0dH/0dHR/9HR0f/r6+v/+vr6/+vr6//6+vr/+/v7//z8/P/7+/v/+/v7//v7 + +//7+/v/+/v7//r6+v/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//z8/P/7+/v/+/v7//v7 + +//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+bm5v/6+vr/7e3t//j4+P/x8fH/8fHx//Hx + 8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx/+Hh + 7f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/x8fH/8fHx//Hx8f/+/v7/5ubm//r6+v/s7Oz/9/f3/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//o5dn/wapM/6+PDv/AqUz/6OXX/+/v + 7//v7+//zMzk/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v/l5eX/+vr6/+rq + 6v/39/f/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/72kP/+tjAb/rYwG/62M + Bv+7ojv/7+/v/+/v7//IyOP/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+/+Xl + 5f/6+vr/6urq//f39//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//rYwG/62M + Bv+tjAb/rYwG/62MBv/u7ez/7+/v/8XF4//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//+/v7/5eXl//r6+v/o6Oj/9/f3/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7ezp/9jM + nv+1mCP/rYwG/62MBv+tjAb/tJci/9jMnv/u7ev/wcHh/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v//7+/v/l5eX/+vr6/+jo6P/39/f/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7u7u/9vR + qf/YzqL/7u7u/9/YuP+ylBr/rYwG/7KUGf/f2Lr/7u7t/9jNof+8s6P/7u7u/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+///v7+/+Tk5P/6+vr/5+fn//f39//v7+//7u7u/86+e/+wkRP/uZ4x/9LE + i//VyZf/7Orl/+/v7//v7+//7+/v/+7u7v/t7Of/7u7u/+/v7//v7+//7+/v/7a01P/WyZj/0sWN/7me + Mv+wkBL/zb16/+7u7v/v7+//7+/v/+/v7//+/v7/5eXl//r6+v/n5+f/9/f3/+/v7//Sw4n/rYwG/62M + Bv+tjAb/rY0J/+ro3//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//tLTd/+vp + 4f+ujgz/rYwG/62MBv+tjAb/0sOK/+/v7//v7+//7+/v//7+/v/l5eX/+vr6/+bm5v/39/f/7+/v/8Ks + Uf+tjAb/rYwG/62MBv+tjAb/18uc/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7/+vr9z/1cmW/62MBv+tjAb/rYwG/62MBv/Cq1H/7+/v/+/v7//v7+///v7+/+Xl5f/6+vr/5+fn//f3 + 9//v7+//y7lx/62MBv+tjAb/rYwG/62MBv/KxcX/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/6ur2//m4tD/rYwG/62MBv+tjAb/rYwG/8y6c//v7+//7+/v/+/v7//+/v7/5ubm//r6 + +v/m5ub/9/f3/+/v7//s6uX/vKI6/62MBv+tjAf/0MGD/7a23v+bm9f/5eXs/+/v7//u7u7/7u7t/+/v + 7//v7+//7+/v/+/v7//v7+//pqba/+/v7//QwYT/rYwI/62MBv+7oTn/7Ork/+/v7//v7+//7+/v//7+ + /v/m5ub/+vr6/+fn5//39/f/7+/v/+/v7//v7+//0MGE/+7t7P/v7+//7+/v/+7u7v+cnNf/rKSp/6+Q + EP+ujQr/3dSx/+/v7//v7+//7+/v/+/v7/+hodn/7+/v/+/v7//t7ev/1ceT/+7u7v/v7+//7u7u/6Cg + 2P/BweL//v7+/+bm5v/6+vr/6Ojo//f39//v7+//7+/v/+/v7//Rwof/7+/v/+/v7//v7+//7+/v/+/v + 7//Fr1v/rYwG/62MBv+8oz3/7+/v/+/v7//v7+//7+/v/52d1//v7+//7+/v/+7u7v/WyZb/7+/v/7S0 + 3v+JidL/4eHr/+/v7//+/v7/5eXl//n5+f/o6Oj/9/f3/+/v7//v7+//7+/v/9HCh//v7+//7+/v/+/v + 7//v7+//7+/v/869e/+tjAb/rYwG/5J/Xf/Pz+b/7+/v/+/v7//v7+//mJjV/+/v7//v7+//7u7u/7Kl + jP+Cgs//v7/h/+/v7//v7+//7+/v//7+/v/l5eX/+vr6/+jo6P/39/f/7+/v/+/v7//v7+//0cKH/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/9THkv/SxIr/7u7u/56e2P+Bgc//5ubs/+/v7/+SktT/7+/v/+Tk + 7P+Cgs//s6aM/+/v7//v7+//7+/v/+/v7//v7+///v7+/+bm5v/6+vr/6Ojo//f39//v7+//7+/v/+/v + 7//Rwof/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+zs7v91dcz/lpbV/4yM + 0/+SktT/eXnN/+vr7f/WyZb/7+/v/+/v7//v7+//7+/v/+/v7//+/v7/5ubm//r6+v/n5+f/9/f3/+/v + 7//v7+7/0MKG/7OVHP++pkT/39i7/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+7u + 7/+IiNH/Dg6v/5qa1v+5ud//RES+/yIhrv+FhdH/7u7u/+/v7//v7+//7+/v//7+/v/l5eX/+vr6/+fn + 5//39/f/7+/v/9PGj/+tjAb/rYwG/62MBv+ujQr/6ujh/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//Q0Ob/d3fM/7S03v+NjdP/oqLZ/wgIrf8GBq3/Bgat/wYGrf+Pj9P/7+/v/+/v7//v7+///v7+/+Xl + 5f/6+vr/5ubm//f39//v7+//wqxS/62MBv+tjAb/rYwG/62MBv/XzJ3/7+/v/+/v7//v7+//7+/v/+/v + 7//q6u3/iIjR/5eX1f/v7+//7+/v/5KS1P+cnNf/Bgat/wYGrf8GBq3/Bgat/1JSwv/v7+//7+/v/+/v + 7//+/v7/5eXl//r6+v/m5ub/9/f3/+/v7//KuG//rYwG/62MBv+tjAb/rYwG/9DChv/q593/7+/v/+/v + 7//v7+//oaHY/4iI0f/q6u3/7+/v/+/v7//v7+//kpDC/7qwo/8GBq3/Bgat/wYGrf8GBq3/Y2PG/+/v + 7//v7+//7+/v//7+/v/l5eX/+vr6/+fn5//39/f/7+/v/+ro3/+5njL/rYwG/62MBv/Mu3X/7+/v/9rR + qP/ZzaH/v7/g/4uL0v/Py83/1cmX/+fk1f/v7+//7u7t/9nNov+poJ7/7+/v/3l5zf8GBq3/Bgat/zIy + uf+3t9//n5/Y/+/v7//v7+///v7+/+Xl5f/6+vr/5+fn//f39//v7+//7+/v/+/v7//n49T/6+nj/+/v + 7//v7+//39/q/5ST0v/Nwpv/zLt1/62MBv+tjAb/rYwG/8y8eP/XzJ7/7u7t/5+f2P/v7+//7+/v/+Pj + 6//U1Of/7+/v/+/v7//v7+//7+/v/+/v7//+/v7/5eXl//r6+v/o6Oj/9/f3/+/v7//v7+//7+/v/+/v + 7//v7+//7u7u/6am2v+oqNr/7+/v/+/v7/+vkBD/rYwG/62MBv+tjAb/sJAR/+/v7//v7+//pKTZ/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+/v/m5ub/+vr6/+fn5//39/f/7+/v/+/v + 7//v7+//7+/v/7W13v+mptr/7Ozu/+/v7//v7+//7+/v/62MBv+tjAb/rYwG/62MBv+tjAb/7u7u/+/v + 7/+pqdv/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+///v7+/+bm5v/6+vr/6urq//f3 + 9//v7+//7+/v/+/v7//BweH/39/q/+/v7//v7+//7+/v/+/v7//v7+//wqtR/62MBv+tjAb/rYwG/8Ww + XP/v7+//7+/v/66u2//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//+/v7/5ubm//r6 + +v/q6ur/9/f3/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//u7u3/zr17/8Ou + Vv/Ov37/7u7t/+/v7//v7+//srLd/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v//7+ + /v/m5ub/+vr6/+vr6//39/f/7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+//7+/v/+/v7//v7+//7+/v/+/v7/+4uN//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v + 7//v7+///v7+/+bm5v/6+vr/7e3t//j4+P/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly8v/y8vL/8vLy//Ly + 8v/y8vL/8vLy//Ly8v/+/v7/5ubm//r6+v/t7e3///////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////m5ub//v7+//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4 + +P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4 + +P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//v7+/8AAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEA + CAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAACtjAb/ro0I/66OCv+vjwz/r5AO/7CQEP+xkRL/spQZ/7OU + Gv+zlRz/tZgi/7mfMv+6nzP/vKI6/7yjPP+9pD7/koBe/7+nRf/Bqkz/wqxR/8OsUv/Erlf/xbBc/8u5 + b//LunL/zLtz/8y7dv/Nu3b/zbx5/869ev/Ovnz/z79//wYGrf8ICK7/Dg6v/yMhrv8yMrr/RUW//1JS + w/9jY8f/dnbN/3h4zf95ec3/sqWN/7Snjf+qoJ//raWp/7qxo/+9s6P/0cGE/9HChv/Sw4f/0sSK/9PE + i//Txo7/1MaP/87DnP/VyJP/1smX/9bKl//Wypj/2Myd/9nOov/b0an/29Kq/97Vsv/g2Lr/k5DD/4KC + 0P+Dg9D/hobR/4iI0f+JidL/iorS/4uL0/+MjNP/jo7U/4+P1P+UlNP/kpLU/5OT1f+Xl9b/mJjW/5mZ + 1v+bm9f/nJzX/52d1/+fn9n/oKDY/6Gh2f+iotn/o6PZ/6Wl2v+mptr/p6fa/6io2/+pqdv/rKzc/66u + 3P+2tdT/sLDd/7Oz3v+0tN7/tbXe/7a23/+3t9//uLjg/7q64P+/v+H/y8bG/9DLzv/R0dH/0tLS/9jY + 2P/n4tH/6OTW/+nm2f/q6N7/6+jf/8DA4v/BweL/wsLi/8XF4//IyOT/zMzl/9DQ5v/U1Of/39/r/+Xl + 5f/m5ub/5+fn/+vp4f/r6uL/7Orj/+zq5f/g4Ov/4uLr/+Li7f/k5Oz/5eXs/+bm7P/o6Oj/6enp/+rq + 6v/r6+v/7ezo/+7t6v/r6+7/7Ozs/+3t7f/u7ez/7u7s/+/u7v/x8fH/8vLy//f39//4+Pj/+fn5//r6 + +v/7+/v//Pz8//39/f//////AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/oXFwcHBwcHBwcHBwcG9vb3BwcHBwcHBwcHBwcG9v + b5SekJ+foaGfoKCgnp+fn6CgnqChoKCfn6Ggn5+fn5+egZ6Vm5qZmpmZmZqampmampmZmpqaiZmampqa + mZqZmaKCnpWbmJmVmZmZlZmVmXQSBBJzmJd8mZWYmJWZmJmZooGej5uZlZmVlZWZlZmYDwAAAA6YmXuV + mZWZmZiYmJiigZ6Pm5WZlZmZmZWZmJcAAAAAAZeYepmVmZiYmZWZlaKBno2bmZWZlZiVmZiUPgoAAAAK + PZR3mJmYmJiVmZiZooCejZuYmZiZlZmYQD6VQgkAB0KZPjCYmJiZmJmVmJWigJ6Nm5iYHAYLNDyGmZiZ + lZGZlZWZYzw2DAUemJmZmaKAno2bmTQAAAABdZmVmJWZmZWZmZVmgwMAAAA0lZiVooCegZuVEwAAAAA9 + lZmYmZWVmZWVmWQ6AAAAABOZl5migJ6Cm5kYAAAAAG2ZmJmYmZmVmZmVYXIAAQAAGZWZlaKBnoKbmJEO + AAIxaFSMlZWYmJmVlZlemTMBAA2GmZiZooKejZuYmJkxlZmZlVYuBQJBlZmZlVqVlZc5lZiYWXiigZ6O + m5mYlTSVmZWZlRYAAA+ZlZWZVpmZlTqZZkeHlaKBno6bmJiZM5iYmZiZHgAAEH2ZmZVTlZiZK0V4mZmZ + ooCejpuVmZgzmJmYlZWZOTWVV0SMmVCZi0Qsl5WVlZWigZ6Om5mYmDOYmJiZmZWZlZmVkyhRS1AplDyZ + mZmZmaKBno2blZgyCRFCmJWVmZWZlZmZmEciVGslI0WYlZWVooCegpuZNwAAAAODmZmVmZWZl30qaExb + ISAgIE2ZmZmigJ6Cm5UUAAAAAD2YlZmVmZNHUpWVT1UgICAgJpeVlaKAnoKbmRcAAAAAMnWZl5laR5OV + mZlDLyAgICAnmZmZooGegpuVdQsBARqZQD53Sm48c5mVPi2VKSAgJGdWlZWigZ6Nm5mZmXKFlZh/Tjga + AAABHD2YWJmZin6ZmZmZmaKBno2blZWVmZiZX12VlQUAAAAGmZVclZWVlZWVlZWVooGejZuZmZmYZ12V + mZmZAQAAAACYmWCZmZmZmZmZmZmigp6Pm5WVlXd/lZWVlZUTAAAAFpWVYpWVlZWVlZWVlaKCno+bmZmZ + mZmZmZmZmZceFR+YmZllmZmZmZmZmZmZooKelJuYmJiYmJiYmJiYmJiYmJiYmGqYmJiYmJiYmJiigp6V + nZqampqampqampqampqampqampqampqampqamqKBn5WioqKioqKioqKioqKioqKioqKioqKioqKioqKi + ooKinZycnJycnJycnJycnJycnJycnJycnJycnJycnJycoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAABAAAAAgAAAAAQAgAAAA + AAAACAAAAAAAAAAAAAAAAAAAAAAAAOPj4//T09P/1dXV/9TU1P/U1NT/1NTU/9TU1P/U1NT/1NTU/9XV + 1f/U1NT/1NTU/9TU1P/U1NT/09PT/9DQ0P/o6Oj/7+/v/+/v7//v7+//7+/v/+/v7//v7+//08aO/+7t + 6//v7+//4eHr/+/v7//v7+//7+/v/+/v7//s7Oz/5eXl/+/v7//v7+//7+/v/+/v7//v7+//0MGE/62M + Bv+tjAb/7+/v/97e6v/v7+//7+/v/+/v7//v7+//6+vr/+Pj4//v7+//7+/v/+/v7//v7+//5N7I/+DZ + vv+tjAb/r44M/9XJlv/b2+j/7+/v/+/v7//v7+//7+/v/+rq6v/i4uL/7+/v/8WxX/+tjAb/y7lx/+/v + 7//v7+//7+/v/+/v7//v7+//2Njn/66NCv+tjAb/7+/v/+/v7//r6+v/4eHh/+/v7/+tjAb/rYwG/7me + Mf/v7+//7+/v/+/v7//v7+//7+/v/9fX5/+tjAb/rYwG/+bhz//v7+//6+vr/+Hh4f/v7+//7Ork/76l + Qv/v7+7/kZHU/+Havv/e1rT/7+/v/+/v7//S0uf/2M2h/86+ff/v7+//xcXj/+zs7P/j4+P/7+/v/+/v + 7//Rwof/7+/v/+/v7/+tjAb/rYwG/+/v7//v7+//z8/m/+/v7//t7e7/np7Y/+/v7//r6+v/5OTk/+/v + 7//v7+//0cKH/+/v7//v7+//7+/v/+/v7//FxeP/wMDh/8zM5f93d83/7+/v/+/v7//v7+//7Ozs/+Li + 4v/v7+//ybZp/62MBv/f17f/7+/v/+/v7//v7+//7+/v/4GBz/9tbcr/DQ2u/wYGrf/v7+//7+/v/+rq + 6v/h4eH/7+/v/62MBv+tjAb/tpkl/+/v7//v7+//yMjj/8XF4//v7+//0NDm/wYGrf8GBq3/zMzk/+/v + 7//r6+v/4uLi/+/v7//q6OD/xbBd/+7u7v/l4M7/lJG9/7qfNP/b0ar/1ciV/9bW5/+amtf/cnLL/+np + 7f/v7+//6+vr/+Pj4//v7+//7+/v/+/v7/+amtf/7+/v/869ev+tjAb/rYwG/+/v7//b2+n/7+/v/+/v + 7//v7+//7+/v/+zs7P/l5eX/7+/v/+/v7//u7u7/7+/v/+/v7//u7ez/rYwG/8CpS//v7+//4eHr/+/v + 7//v7+//7+/v/+/v7//s7Oz/6Ojo/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+/v7//v7+//7+/v/+vr + 7v/v7+//7+/v/+/v7//v7+//7Ozs//Hx8f/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4//j4 + +P/4+Pj/+Pj4//j4+P/4+Pj/+Pj4/+7u7v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAABAAAAAgAAAAAQAIAAAAAAAAAgAAAAAAAAAA + AAAAAAAAAAAAAK2MBv+ujQn/ro4L/7GSE/+xkhT/s5Ub/6qPKv+2mib/up80/7qgNf+7oTj/vaQ//7+q + XP/Bqkz/wKtc/8ayYP/LuXD/zLp0/8+/ff8GBq3/LS22/0NDvv92ds3/f3/P/7mysf++uLj/0MGD/9LE + if/Ux5H/1ceQ/9fLmv/Pxab/2tCm/9vRqf/g2bv/4dm8/+Lbv/+fnMT/lJTV/6Wl2v+oqNv/qanb/7Oz + 3v+4uN//u7vg/76+4f+/v+L/ysTC/8bCyP/Lx8v/2tPA/8rI1f/Z1dD/19XZ/+Pcwv/j3sb/5N7H/+Tf + yf/m4tH/6OTW/+jl2P/BweL/w8Pi/8XF4//KyuT/y8vl/83N5f/Q0Ob/19fo/9vb6f/d3er/5ubm/+fn + 5//g4Ov/4eHr/+Pj7f/k5O3/5+ft/+jo6P/u7ev/7+7r/+zs7v/u7u3/7+/u//Dw8P/x8fH/8vLy//Pz + 8//09PT/9vb2//j4+P/8/Pz/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAA + AP8AAAD/AAAA/wAAAP9SR0hHSEdHR0dIR0dHR0dOWFhUVFRUTxw2VExUVFRUVlZXVVNUUxsABFNFVFRU + VFZVV1FUUjkeAgs5NVJSU1JWVVcSBBpSVFJSUTEIBztUVlVXCAAMVFNUVFQwAQAhVFZVVyQPND0fIFNT + QB4QT0JWVVdUIlRUCgZNVD9SGSxRVlVXUyJSUjw7LisoKjJUVFZVVxoFIVRUU00oFhUURFNWVFcIAA1P + Uz0tUyUTEydUVlVXIhA6LxgOHDczJhdBUlZVV1RKQEYSAAFUQVRUVFRWVldTSVNTOAoRU0NTU1NTV1dY + VVVVVVVVVVVLVVVVVVZaW1tbW1tbW1tbW1tbW1tZAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== + + + \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs new file mode 100644 index 0000000000..2597bd6636 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.Designer.cs @@ -0,0 +1,246 @@ +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + partial class CreateNewDatasetProviderConfigurationUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + label2 = new System.Windows.Forms.Label(); + label3 = new System.Windows.Forms.Label(); + label4 = new System.Windows.Forms.Label(); + tbName = new System.Windows.Forms.TextBox(); + tbUrl = new System.Windows.Forms.TextBox(); + tbOrganisationId = new System.Windows.Forms.TextBox(); + cbCredentials = new System.Windows.Forms.ComboBox(); + btnSave = new System.Windows.Forms.Button(); + cbImportCatalogues = new System.Windows.Forms.CheckBox(); + cbIncludeInternal = new System.Windows.Forms.CheckBox(); + cbImportProjectSpecific = new System.Windows.Forms.CheckBox(); + cbImportDeprecated = new System.Windows.Forms.CheckBox(); + aiImportAll = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + aiInternal = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + aiProjectSpecific = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + aiDeprecated = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(57, 76); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(104, 15); + label1.TabIndex = 0; + label1.Text = "URL/ID:"; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(119, 37); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(42, 15); + label2.TabIndex = 1; + label2.Text = "Name:"; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(26, 116); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(135, 15); + label3.TabIndex = 2; + label3.Text = "Data Access Credentials:"; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(69, 152); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(92, 15); + label4.TabIndex = 3; + label4.Text = "Organisation ID:"; + // + // tbName + // + tbName.Location = new System.Drawing.Point(169, 34); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(370, 23); + tbName.TabIndex = 4; + tbName.TextChanged += ValidateForm; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(169, 73); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(370, 23); + tbUrl.TabIndex = 5; + tbUrl.TextChanged += ValidateForm; + // + // tbOrganisationId + // + tbOrganisationId.Location = new System.Drawing.Point(169, 149); + tbOrganisationId.Name = "tbOrganisationId"; + tbOrganisationId.Size = new System.Drawing.Size(370, 23); + tbOrganisationId.TabIndex = 6; + tbOrganisationId.TextChanged += ValidateForm; + // + // cbCredentials + // + cbCredentials.FormattingEnabled = true; + cbCredentials.Location = new System.Drawing.Point(169, 108); + cbCredentials.Name = "cbCredentials"; + cbCredentials.Size = new System.Drawing.Size(208, 23); + cbCredentials.TabIndex = 7; + cbCredentials.SelectedIndexChanged += ValidateForm; + // + // btnSave + // + btnSave.Enabled = false; + btnSave.Location = new System.Drawing.Point(464, 318); + btnSave.Name = "btnSave"; + btnSave.Size = new System.Drawing.Size(75, 23); + btnSave.TabIndex = 8; + btnSave.Text = "Save"; + btnSave.UseVisualStyleBackColor = true; + btnSave.Click += Save; + // + // cbImportCatalogues + // + cbImportCatalogues.AutoSize = true; + cbImportCatalogues.Location = new System.Drawing.Point(337, 188); + cbImportCatalogues.Name = "cbImportCatalogues"; + cbImportCatalogues.Size = new System.Drawing.Size(202, 19); + cbImportCatalogues.TabIndex = 9; + cbImportCatalogues.Text = "Import All Extractable Catalogues"; + cbImportCatalogues.UseVisualStyleBackColor = true; + // + // cbIncludeInternal + // + cbIncludeInternal.AutoSize = true; + cbIncludeInternal.Location = new System.Drawing.Point(337, 213); + cbIncludeInternal.Name = "cbIncludeInternal"; + cbIncludeInternal.Size = new System.Drawing.Size(184, 19); + cbIncludeInternal.TabIndex = 10; + cbIncludeInternal.Text = "Import All Internal Catalogues"; + cbIncludeInternal.UseVisualStyleBackColor = true; + // + // cbImportProjectSpecific + // + cbImportProjectSpecific.AutoSize = true; + cbImportProjectSpecific.Location = new System.Drawing.Point(337, 238); + cbImportProjectSpecific.Name = "cbImportProjectSpecific"; + cbImportProjectSpecific.Size = new System.Drawing.Size(225, 19); + cbImportProjectSpecific.TabIndex = 11; + cbImportProjectSpecific.Text = "Import All Project Sepcific Catalogues"; + cbImportProjectSpecific.UseVisualStyleBackColor = true; + // + // cbImportDeprecated + // + cbImportDeprecated.AutoSize = true; + cbImportDeprecated.Location = new System.Drawing.Point(337, 263); + cbImportDeprecated.Name = "cbImportDeprecated"; + cbImportDeprecated.Size = new System.Drawing.Size(204, 19); + cbImportDeprecated.TabIndex = 12; + cbImportDeprecated.Text = "Import All Deprecated Catalogues"; + cbImportDeprecated.UseVisualStyleBackColor = true; + // + // aiImportAll + // + aiImportAll.Location = new System.Drawing.Point(542, 188); + aiImportAll.Name = "aiImportAll"; + aiImportAll.Size = new System.Drawing.Size(20, 20); + aiImportAll.TabIndex = 13; + // + // aiInternal + // + aiInternal.Location = new System.Drawing.Point(521, 213); + aiInternal.Name = "aiInternal"; + aiInternal.Size = new System.Drawing.Size(20, 20); + aiInternal.TabIndex = 14; + // + // aiProjectSpecific + // + aiProjectSpecific.Location = new System.Drawing.Point(559, 237); + aiProjectSpecific.Name = "aiProjectSpecific"; + aiProjectSpecific.Size = new System.Drawing.Size(20, 20); + aiProjectSpecific.TabIndex = 15; + // + // aiDeprecated + // + aiDeprecated.Location = new System.Drawing.Point(542, 263); + aiDeprecated.Name = "aiDeprecated"; + aiDeprecated.Size = new System.Drawing.Size(20, 20); + aiDeprecated.TabIndex = 16; + // + // CreateNewJiraConfigurationUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(800, 450); + Controls.Add(aiDeprecated); + Controls.Add(aiProjectSpecific); + Controls.Add(aiInternal); + Controls.Add(aiImportAll); + Controls.Add(cbImportDeprecated); + Controls.Add(cbImportProjectSpecific); + Controls.Add(cbIncludeInternal); + Controls.Add(cbImportCatalogues); + Controls.Add(btnSave); + Controls.Add(cbCredentials); + Controls.Add(tbOrganisationId); + Controls.Add(tbUrl); + Controls.Add(tbName); + Controls.Add(label4); + Controls.Add(label3); + Controls.Add(label2); + Controls.Add(label1); + Name = "CreateNewJiraConfigurationUI"; + Text = "Create New Dataset Provider Configuration"; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.TextBox tbName; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.TextBox tbOrganisationId; + private System.Windows.Forms.ComboBox cbCredentials; + private System.Windows.Forms.Button btnSave; + private System.Windows.Forms.CheckBox cbImportCatalogues; + private System.Windows.Forms.CheckBox cbIncludeInternal; + private System.Windows.Forms.CheckBox cbImportProjectSpecific; + private System.Windows.Forms.CheckBox cbImportDeprecated; + private SimpleControls.AdditionalInfomationUI aiImportAll; + private SimpleControls.AdditionalInfomationUI aiInternal; + private SimpleControls.AdditionalInfomationUI aiProjectSpecific; + private SimpleControls.AdditionalInfomationUI aiDeprecated; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs new file mode 100644 index 0000000000..c870f95970 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/CreateNewDatasetProviderConfigurationUI.cs @@ -0,0 +1,82 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + public partial class CreateNewDatasetProviderConfigurationUI : RDMPForm + { + private readonly IActivateItems _activator; + private readonly Type _providerType; + public CreateNewDatasetProviderConfigurationUI(IActivateItems activator, Type providerType) + { + InitializeComponent(); + _activator = activator; + _providerType = providerType; + var dataAccessCredentials = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects(); + cbCredentials.Items.Clear(); + cbCredentials.Items.AddRange(dataAccessCredentials); + } + + private void DisableSave() + { + btnSave.Enabled = false; + } + + private void ValidateForm(object sender, EventArgs e) + { + + if (cbCredentials.SelectedItem is null) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbName.Text)) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbUrl.Text)) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbOrganisationId.Text)) + { + DisableSave(); + return; + } + btnSave.Enabled = true; + } + + private void Save(object sender, EventArgs e) + { + var config = new DatasetProviderConfiguration(_activator.RepositoryLocator.CatalogueRepository, tbName.Text, _providerType.FullName, tbUrl.Text, ((DataAccessCredentials)cbCredentials.SelectedItem).ID, tbOrganisationId.Text); + config.SaveToDatabase(); + _activator.Publish(config); + + if (cbImportCatalogues.Checked || cbImportProjectSpecific.Checked || cbIncludeInternal.Checked || cbImportDeprecated.Checked) + { + var provider = config.GetProviderInstance(_activator); + var autoUpdate = _activator.YesNo("Do you want these dataset to be updated automatically?", "Automatically Update Datasets?"); + var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(_activator, provider, cbImportCatalogues.Checked, cbIncludeInternal.Checked, cbImportProjectSpecific.Checked, cbImportDeprecated.Checked, autoUpdate); + cmd.Execute(); + } + + Close(); + _activator.Show($"Dataset Provider '{tbName.Text}' has successfully been created"); + } + } + +} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs new file mode 100644 index 0000000000..f6eeac2276 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.Designer.cs @@ -0,0 +1,131 @@ +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + partial class ImportExistingDatasetUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + btnCreate = new System.Windows.Forms.Button(); + btnCancel = new System.Windows.Forms.Button(); + tbUrl = new System.Windows.Forms.TextBox(); + lblError = new System.Windows.Forms.Label(); + label2 = new System.Windows.Forms.Label(); + cbProviders = new System.Windows.Forms.ComboBox(); + label1 = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // btnCreate + // + btnCreate.Location = new System.Drawing.Point(550, 108); + btnCreate.Name = "btnCreate"; + btnCreate.Size = new System.Drawing.Size(75, 23); + btnCreate.TabIndex = 14; + btnCreate.Text = "Create"; + btnCreate.UseVisualStyleBackColor = true; + btnCreate.Click += btnCreate_Click; + // + // btnCancel + // + btnCancel.Location = new System.Drawing.Point(459, 108); + btnCancel.Name = "btnCancel"; + btnCancel.Size = new System.Drawing.Size(75, 23); + btnCancel.TabIndex = 15; + btnCancel.Text = "Cancel"; + btnCancel.UseVisualStyleBackColor = true; + btnCancel.Click += btnCancel_Click; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(74, 56); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(551, 23); + tbUrl.TabIndex = 17; + // + // lblError + // + lblError.AutoSize = true; + lblError.ForeColor = System.Drawing.Color.Red; + lblError.Location = new System.Drawing.Point(23, 95); + lblError.Name = "lblError"; + lblError.Size = new System.Drawing.Size(0, 15); + lblError.TabIndex = 18; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(14, 20); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(54, 15); + label2.TabIndex = 19; + label2.Text = "Provider:"; + // + // cbProviders + // + cbProviders.FormattingEnabled = true; + cbProviders.Location = new System.Drawing.Point(74, 17); + cbProviders.Name = "cbProviders"; + cbProviders.Size = new System.Drawing.Size(345, 23); + cbProviders.TabIndex = 20; + cbProviders.SelectedIndexChanged += cbProviders_SelectedIndexChanged; + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(37, 59); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(37, 15); + label1.TabIndex = 16; + label1.Text = "UUID:"; + // + // ImportExistingPureDatasetUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(667, 158); + Controls.Add(cbProviders); + Controls.Add(label2); + Controls.Add(lblError); + Controls.Add(tbUrl); + Controls.Add(label1); + Controls.Add(btnCancel); + Controls.Add(btnCreate); + Name = "ImportExistingPureDatasetUI"; + Text = "Import Existing Dataset"; + Load += ImportExistingPureDatasetUI_Load; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + private System.Windows.Forms.Button btnCreate; + private System.Windows.Forms.Button btnCancel; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.Label lblError; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.ComboBox cbProviders; + private System.Windows.Forms.Label label1; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs new file mode 100644 index 0000000000..6e84677ea2 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/ImportExistingDatasetUI.cs @@ -0,0 +1,67 @@ +using Microsoft.Data.SqlClient; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.UI.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Datasets +{ + public partial class ImportExistingDatasetUI: RDMPForm + { + private readonly IActivateItems _activator; + private readonly Type _providerType; + public ImportExistingDatasetUI(IActivateItems activator, Type providerType) : base(activator) + { + _activator = activator; + _providerType = providerType; + InitializeComponent(); + var configs = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Type", _providerType.ToString()); + cbProviders.Items.AddRange(configs); + } + + private void ImportExistingPureDatasetUI_Load(object sender, EventArgs e) + { + + } + + private void btnCancel_Click(object sender, EventArgs e) + { + Dispose(); + } + + private void btnCreate_Click(object sender, EventArgs e) + { + if (_activator.YesNo("Please confirm you wish to import this Dataset", "Import Dataset")) + { + try + { + var provider = (cbProviders.SelectedItem as DatasetProviderConfiguration).GetProviderInstance(_activator); + provider.AddExistingDataset(null, tbUrl.Text); + Close(); + Dispose(); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + lblError.Text = "Unable to locate dataset. Ensure UUID is correct."; + lblError.Visible = true; + return; + } + } + } + + private void cbProviders_SelectedIndexChanged(object sender, EventArgs e) + { + } + } +} + diff --git a/Rdmp.UI/SimpleDialogs/NewfindUI.cs b/Rdmp.UI/SimpleDialogs/NewfindUI.cs index 8b49fba779..4d7b572fdb 100644 --- a/Rdmp.UI/SimpleDialogs/NewfindUI.cs +++ b/Rdmp.UI/SimpleDialogs/NewfindUI.cs @@ -369,12 +369,8 @@ private void BuildToolStripForDatabaseObjects(RDMPCollection focusedCollection) v => UserSettings.ShowInternalCatalogues = v, "I", "Include Internal"); AddUserSettingCheckbox(() => UserSettings.ShowDeprecatedCatalogues, v => UserSettings.ShowDeprecatedCatalogues = v, "D", "Include Deprecated"); - AddUserSettingCheckbox(() => UserSettings.ShowColdStorageCatalogues, - v => UserSettings.ShowColdStorageCatalogues = v, "C", "Include Cold Storage"); AddUserSettingCheckbox(() => UserSettings.ShowProjectSpecificCatalogues, v => UserSettings.ShowProjectSpecificCatalogues = v, "P", "Include Project Specific"); - AddUserSettingCheckbox(() => UserSettings.ShowNonExtractableCatalogues, - v => UserSettings.ShowNonExtractableCatalogues = v, "E", "Include Extractable"); } } diff --git a/Rdmp.UI/SimpleDialogs/Reports/MetadataReportUI.cs b/Rdmp.UI/SimpleDialogs/Reports/MetadataReportUI.cs index 331467f5cb..8cd2f2c414 100644 --- a/Rdmp.UI/SimpleDialogs/Reports/MetadataReportUI.cs +++ b/Rdmp.UI/SimpleDialogs/Reports/MetadataReportUI.cs @@ -66,7 +66,7 @@ private void btnGenerateReport_Click(object sender, EventArgs e) IEnumerable toReportOn = _catalogues; if (rbAllCatalogues.Checked) - toReportOn = toReportOn.Where(c => !c.IsInternalDataset && !c.IsColdStorageDataset && !c.IsDeprecated) + toReportOn = toReportOn.Where(c => !c.IsInternalDataset && !c.IsDeprecated) .ToArray(); else if (_cataloguesToRun == null || !_catalogues.Any()) return; diff --git a/Rdmp.UI/SimpleDialogs/SelectDialog.cs b/Rdmp.UI/SimpleDialogs/SelectDialog.cs index d508c4c946..b02f7f82e6 100644 --- a/Rdmp.UI/SimpleDialogs/SelectDialog.cs +++ b/Rdmp.UI/SimpleDialogs/SelectDialog.cs @@ -169,7 +169,7 @@ public SelectDialog(DialogArgs args, IActivateItems activator, IEnumerable to if (IsDatabaseObjects()) { - _allObjects = toSelectFrom.ToArray(); + _allObjects = toSelectFrom.Distinct().ToArray(); _searchables = _allObjects.Cast() .ToDictionary(k => k, activator.CoreChildProvider.GetDescendancyListIfAnyFor); _usefulPropertyFinder = new AttributePropertyFinder(_searchables.Keys); @@ -188,7 +188,7 @@ public SelectDialog(DialogArgs args, IActivateItems activator, IEnumerable to } else { - _allObjects = toSelectFrom.ToArray(); + _allObjects = toSelectFrom.Distinct().ToArray(); // don't bother with the tool strip because its not database objects so we can't filter by ID/type etc Controls.Remove(toolStrip1); @@ -448,12 +448,8 @@ private void BuildToolStripForDatabaseObjects(RDMPCollection focusedCollection) v => UserSettings.ShowInternalCatalogues = v, "I", "Include Internal"); AddUserSettingCheckbox(() => UserSettings.ShowDeprecatedCatalogues, v => UserSettings.ShowDeprecatedCatalogues = v, "D", "Include Deprecated"); - AddUserSettingCheckbox(() => UserSettings.ShowColdStorageCatalogues, - v => UserSettings.ShowColdStorageCatalogues = v, "C", "Include Cold Storage"); AddUserSettingCheckbox(() => UserSettings.ShowProjectSpecificCatalogues, v => UserSettings.ShowProjectSpecificCatalogues = v, "P", "Include Project Specific"); - AddUserSettingCheckbox(() => UserSettings.ShowNonExtractableCatalogues, - v => UserSettings.ShowNonExtractableCatalogues = v, "E", "Include Extractable"); } } diff --git a/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs b/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs index 4a8409fc1a..961a2a1718 100644 --- a/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs @@ -100,6 +100,7 @@ private void InitializeComponent() userSettingsToolTips = new System.Windows.Forms.ToolTip(components); tbFind = new System.Windows.Forms.TextBox(); label14 = new System.Windows.Forms.Label(); + cbFlatLogs = new System.Windows.Forms.CheckBox(); groupBox1.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)olvErrorCodes).BeginInit(); groupBox2.SuspendLayout(); @@ -454,7 +455,7 @@ private void InitializeComponent() // cbAutoRunSqlQueries // cbAutoRunSqlQueries.AutoSize = true; - cbAutoRunSqlQueries.Location = new System.Drawing.Point(4, 102); + cbAutoRunSqlQueries.Location = new System.Drawing.Point(7, 104); cbAutoRunSqlQueries.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); cbAutoRunSqlQueries.Name = "cbAutoRunSqlQueries"; cbAutoRunSqlQueries.Size = new System.Drawing.Size(138, 19); @@ -489,7 +490,7 @@ private void InitializeComponent() // cbNewFind // cbNewFind.AutoSize = true; - cbNewFind.Location = new System.Drawing.Point(6, 97); + cbNewFind.Location = new System.Drawing.Point(7, 97); cbNewFind.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); cbNewFind.Name = "cbNewFind"; cbNewFind.Size = new System.Drawing.Size(98, 19); @@ -631,7 +632,7 @@ private void InitializeComponent() cbPromptFilterRename.AutoSize = true; cbPromptFilterRename.Checked = true; cbPromptFilterRename.CheckState = System.Windows.Forms.CheckState.Checked; - cbPromptFilterRename.Location = new System.Drawing.Point(6, 89); + cbPromptFilterRename.Location = new System.Drawing.Point(7, 89); cbPromptFilterRename.Name = "cbPromptFilterRename"; cbPromptFilterRename.Size = new System.Drawing.Size(222, 19); cbPromptFilterRename.TabIndex = 4; @@ -660,6 +661,7 @@ private void InitializeComponent() // // groupBox7 // + groupBox7.Controls.Add(cbFlatLogs); groupBox7.Controls.Add(label16); groupBox7.Controls.Add(tbLogLocation); groupBox7.Controls.Add(label15); @@ -851,6 +853,17 @@ private void InitializeComponent() label14.TabIndex = 26; label14.Text = "Find Setting:"; // + // cbFlatLogs + // + cbFlatLogs.AutoSize = true; + cbFlatLogs.Location = new System.Drawing.Point(7, 257); + cbFlatLogs.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); + cbFlatLogs.Name = "cbFlatLogs"; + cbFlatLogs.Size = new System.Drawing.Size(140, 19); + cbFlatLogs.TabIndex = 29; + cbFlatLogs.Text = "Show Logs as Flat List"; + cbFlatLogs.UseVisualStyleBackColor = true; + // // UserSettingsFileUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); @@ -964,5 +977,6 @@ private void InitializeComponent() private System.Windows.Forms.TextBox tbWebhookUrl; private System.Windows.Forms.Label label18; private System.Windows.Forms.TextBox tbWebhookUsername; + private System.Windows.Forms.CheckBox cbFlatLogs; } } \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs b/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs index 75a35c30e1..84d81b5e2c 100644 --- a/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs +++ b/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs @@ -108,6 +108,7 @@ public UserSettingsFileUI(IActivateItems activator) RegisterCheckbox(cbUseAliasInsteadOfTransformInGroupByAggregateGraphs, nameof(UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs)); RegisterCheckbox(cbUseLocalFileSystem, nameof(UserSettings.UseLocalFileSystem)); + RegisterCheckbox(cbFlatLogs, nameof(UserSettings.DefaultLogViewFlat)); AddTooltip(label7, nameof(UserSettings.CreateDatabaseTimeout)); AddTooltip(tbCreateDatabaseTimeout, nameof(UserSettings.CreateDatabaseTimeout)); AddTooltip(label13, nameof(UserSettings.ArchiveTriggerTimeout)); diff --git a/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs b/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs index 558ab5a393..dc12479c10 100644 --- a/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs @@ -142,7 +142,6 @@ public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) { Common.Activator = Activator; var descendancy = Activator.CoreChildProvider.GetDescendancyListIfAnyFor(e.Object); - //if publish event was for a child of the cic (_cic is in the objects descendancy i.e. it sits below our cic) if (descendancy != null && descendancy.Parents.Contains(Common.Configuration)) { @@ -177,7 +176,10 @@ public override void SetDatabaseObject(IActivateItems activator, CohortIdentific gbCicInfo.Text = $"Name: {databaseObject.Name}"; tbDescription.Text = $"Description: {databaseObject.Description}"; ticket.TicketText = databaseObject.Ticket; - + if (databaseObject.IsTemplate) + { + version.Visible = false; + } if (_commonFunctionality == null) { activator.RefreshBus.Subscribe(this); @@ -216,12 +218,14 @@ public override void SetDatabaseObject(IActivateItems activator, CohortIdentific CommonFunctionality.AddToMenu( new ExecuteCommandShowXmlDoc(activator, "CohortIdentificationConfiguration.QueryCachingServer_ID", "Query Caching"), "Help (What is Query Caching)"); - CommonFunctionality.Add( - new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration(activator, null).SetTarget( - databaseObject), - "Commit Cohort", - activator.CoreIconProvider.GetImage(RDMPConcept.ExtractableCohort, OverlayKind.Add)); - + if (!databaseObject.IsTemplate) + { + CommonFunctionality.Add( + new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration(activator, null).SetTarget( + databaseObject), + "Commit Cohort", + activator.CoreIconProvider.GetImage(RDMPConcept.ExtractableCohort, OverlayKind.Add)); + } foreach (var c in _timeoutControls.GetControls()) CommonFunctionality.Add(c); diff --git a/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.Designer.cs b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.Designer.cs new file mode 100644 index 0000000000..c19d29eedb --- /dev/null +++ b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.Designer.cs @@ -0,0 +1,83 @@ +namespace Rdmp.UI.SubComponents +{ + partial class CreateExternalDatasetDialog + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + groupBox3 = new System.Windows.Forms.GroupBox(); + cbPovider = new System.Windows.Forms.ComboBox(); + button1 = new System.Windows.Forms.Button(); + groupBox3.SuspendLayout(); + SuspendLayout(); + // + // groupBox3 + // + groupBox3.Controls.Add(cbPovider); + groupBox3.Location = new System.Drawing.Point(12, 8); + groupBox3.Name = "groupBox3"; + groupBox3.Size = new System.Drawing.Size(200, 60); + groupBox3.TabIndex = 2; + groupBox3.TabStop = false; + groupBox3.Text = "Dataset Provider"; + // + // cbPovider + // + cbPovider.FormattingEnabled = true; + cbPovider.Location = new System.Drawing.Point(6, 22); + cbPovider.Name = "cbPovider"; + cbPovider.Size = new System.Drawing.Size(188, 23); + cbPovider.TabIndex = 1; + // + // button1 + // + button1.Location = new System.Drawing.Point(123, 74); + button1.Name = "button1"; + button1.Size = new System.Drawing.Size(89, 23); + button1.TabIndex = 3; + button1.Text = "Create"; + button1.UseVisualStyleBackColor = true; + button1.Click += button1_Click; + // + // CreateExternalDatasetDialog + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(303, 149); + Controls.Add(button1); + Controls.Add(groupBox3); + Name = "CreateExternalDatasetDialog"; + Text = "Create Dataset"; + groupBox3.ResumeLayout(false); + ResumeLayout(false); + } + + #endregion + private System.Windows.Forms.GroupBox groupBox3; + private System.Windows.Forms.ComboBox cbPovider; + private System.Windows.Forms.Button button1; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs new file mode 100644 index 0000000000..bd07409e71 --- /dev/null +++ b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs @@ -0,0 +1,47 @@ +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data; +using Rdmp.UI.ItemActivation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SubComponents +{ + public partial class CreateExternalDatasetDialog : Form + { + IActivateItems _activator; + private Catalogue _catalogue; + + + public CreateExternalDatasetDialog() + { + InitializeComponent(); + } + + public void Setup(IActivateItems activator, Catalogue catalogue) + { + _activator = activator; + _catalogue = catalogue; + cbPovider.Items.AddRange(_activator.RepositoryLocator.CatalogueRepository.GetAllObjects()); + } + + private void button1_Click(object sender, EventArgs e) + { + var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(_activator); + var dataset = provider.Create(_catalogue); + var ds = provider.AddExistingDatasetWithReturn(null, dataset.GetID()); + if (!_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Dataset_ID", ds.ID).Where(l => l.Catalogue.ID == _catalogue.ID).Any()) + { + var linkage = new CatalogueDatasetLinkage(_activator.RepositoryLocator.CatalogueRepository, _catalogue, ds); + linkage.SaveToDatabase(); + } + Close(); + } + } +} diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs deleted file mode 100644 index b251b593bc..0000000000 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.Designer.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace Rdmp.UI.SubComponents; - -partial class DatasetConfigurationUI -{ - /// - /// Required designer variable. - /// - private System.ComponentModel.IContainer components = null; - - /// - /// Clean up any resources being used. - /// - /// true if managed resources should be disposed; otherwise, false. - protected override void Dispose(bool disposing) - { - if (disposing && (components != null)) - { - components.Dispose(); - } - base.Dispose(disposing); - } - - #region Windows Form Designer generated code - - /// - /// Required method for Designer support - do not modify - /// the contents of this method with the code editor. - /// - private void InitializeComponent() - { - label1 = new System.Windows.Forms.Label(); - tbName = new System.Windows.Forms.TextBox(); - tbDOI = new System.Windows.Forms.TextBox(); - tbSource = new System.Windows.Forms.TextBox(); - label2 = new System.Windows.Forms.Label(); - label3 = new System.Windows.Forms.Label(); - lblDatasetUsage = new System.Windows.Forms.Label(); - SuspendLayout(); - // - // label1 - // - label1.AutoSize = true; - label1.Location = new System.Drawing.Point(24, 40); - label1.Name = "label1"; - label1.Size = new System.Drawing.Size(86, 15); - label1.TabIndex = 0; - label1.Text = "Dataset Name*"; - label1.Click += label1_Click; - // - // tbName - // - tbName.Location = new System.Drawing.Point(27, 65); - tbName.Name = "tbName"; - tbName.Size = new System.Drawing.Size(270, 23); - tbName.TabIndex = 1; - // - // tbDOI - // - tbDOI.Location = new System.Drawing.Point(24, 155); - tbDOI.Name = "tbDOI"; - tbDOI.Size = new System.Drawing.Size(273, 23); - tbDOI.TabIndex = 2; - // - // tbSource - // - tbSource.Location = new System.Drawing.Point(24, 235); - tbSource.Name = "tbSource"; - tbSource.Size = new System.Drawing.Size(273, 23); - tbSource.TabIndex = 3; - // - // label2 - // - label2.AutoSize = true; - label2.Location = new System.Drawing.Point(24, 123); - label2.Name = "label2"; - label2.Size = new System.Drawing.Size(69, 15); - label2.TabIndex = 4; - label2.Text = "Dataset DOI"; - // - // label3 - // - label3.AutoSize = true; - label3.Location = new System.Drawing.Point(28, 202); - label3.Name = "label3"; - label3.Size = new System.Drawing.Size(85, 15); - label3.TabIndex = 5; - label3.Text = "Dataset Source"; - // - // lblDatasetUsage - // - lblDatasetUsage.AutoSize = true; - lblDatasetUsage.Location = new System.Drawing.Point(369, 65); - lblDatasetUsage.Name = "lblDatasetUsage"; - lblDatasetUsage.Size = new System.Drawing.Size(38, 15); - lblDatasetUsage.TabIndex = 8; - lblDatasetUsage.Text = "label5"; - // - // DatasetConfigurationUI - // - AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); - AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - Controls.Add(lblDatasetUsage); - Controls.Add(label3); - Controls.Add(label2); - Controls.Add(tbSource); - Controls.Add(tbDOI); - Controls.Add(tbName); - Controls.Add(label1); - Name = "DatasetConfigurationUI"; - Size = new System.Drawing.Size(800, 450); - ResumeLayout(false); - PerformLayout(); - } - - #endregion - - private System.Windows.Forms.Label label1; - private System.Windows.Forms.TextBox tbName; - private System.Windows.Forms.TextBox tbDOI; - private System.Windows.Forms.TextBox tbSource; - private System.Windows.Forms.Label label2; - private System.Windows.Forms.Label label3; - private System.Windows.Forms.Label lblDatasetUsage; -} diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs b/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs deleted file mode 100644 index dc7315fb89..0000000000 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.cs +++ /dev/null @@ -1,69 +0,0 @@ -using Rdmp.UI.TestsAndSetup.ServicePropogation; -using System; -using System.ComponentModel; -using System.Linq; -using System.Windows.Forms; -using Rdmp.UI.Refreshing; -using Rdmp.Core.Dataset; -using Rdmp.Core.Curation.Data; -using Rdmp.UI.ItemActivation; - -namespace Rdmp.UI.SubComponents; -public partial class DatasetConfigurationUI : DatsetConfigurationUI_Design, IRefreshBusSubscriber -{ - private readonly DatasetConfigurationUICommon _common; - - public DatasetConfigurationUI() - { - InitializeComponent(); - _common = new DatasetConfigurationUICommon(); - } - - public override void SetDatabaseObject(IActivateItems activator, Dataset databaseObject) - { - base.SetDatabaseObject(activator, databaseObject); - _common.Dataset = databaseObject; - - var catalogues = databaseObject.CatalogueRepository - .GetAllObjectsWhere("Dataset_ID", databaseObject.ID).SelectMany(static ci => ci.CatalogueItems) - .Select(static ci => ci.CatalogueName).Distinct().ToList(); - if(catalogues.Count < 1) - { - lblDatasetUsage.Text = "This dataset is not linked to data yet."; - } - else - { - var catalogueString = string.Join(Environment.NewLine, catalogues); - lblDatasetUsage.Text = $"This dataset is used in the following catalogues:{Environment.NewLine}{catalogueString}"; - } - - Bind(tbName, "Text", "Name", static c => c.Name); - Bind(tbDOI, "Text", "DigitalObjectIdentifier", static c => c.DigitalObjectIdentifier); - Bind(tbSource, "Text", "Source", static c => c.Source); - var s = GetObjectSaverButton(); - s.SetupFor(this, databaseObject, activator); - GetObjectSaverButton()?.Enable(false); - - } - - - public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) - { - } - - private void label1_Click(object sender, EventArgs e) - { - - } - - private void label4_Click(object sender, EventArgs e) - { - - } -} -[TypeDescriptionProvider( - typeof(AbstractControlDescriptionProvider))] -public abstract class - DatsetConfigurationUI_Design : RDMPSingleDatabaseObjectControl -{ -} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/DatasetConfigurationUI.resx b/Rdmp.UI/SubComponents/DatasetConfigurationUI.resx deleted file mode 100644 index af32865ec1..0000000000 --- a/Rdmp.UI/SubComponents/DatasetConfigurationUI.resx +++ /dev/null @@ -1,120 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs new file mode 100644 index 0000000000..79b828a1f1 --- /dev/null +++ b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.Designer.cs @@ -0,0 +1,173 @@ +namespace Rdmp.UI.SubComponents +{ + partial class DatasetProviderConfigurationUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Component Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + tbName = new System.Windows.Forms.TextBox(); + btnSave = new System.Windows.Forms.Button(); + tbType = new System.Windows.Forms.TextBox(); + label2 = new System.Windows.Forms.Label(); + tbUrl = new System.Windows.Forms.TextBox(); + label3 = new System.Windows.Forms.Label(); + tbOrgId = new System.Windows.Forms.TextBox(); + label4 = new System.Windows.Forms.Label(); + cbAccessCredentials = new System.Windows.Forms.ComboBox(); + label5 = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(102, 75); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(42, 15); + label1.TabIndex = 0; + label1.Text = "Name:"; + label1.Click += label1_Click; + // + // tbName + // + tbName.Location = new System.Drawing.Point(146, 72); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(252, 23); + tbName.TabIndex = 1; + // + // btnSave + // + btnSave.Location = new System.Drawing.Point(323, 279); + btnSave.Name = "btnSave"; + btnSave.Size = new System.Drawing.Size(75, 23); + btnSave.TabIndex = 2; + btnSave.Text = "Save"; + btnSave.UseVisualStyleBackColor = true; + btnSave.Click += btnSave_Click; + // + // tbType + // + tbType.Enabled = false; + tbType.Location = new System.Drawing.Point(146, 115); + tbType.Name = "tbType"; + tbType.Size = new System.Drawing.Size(252, 23); + tbType.TabIndex = 4; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(102, 118); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(34, 15); + label2.TabIndex = 3; + label2.Text = "Type:"; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(146, 154); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(252, 23); + tbUrl.TabIndex = 6; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(102, 157); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(31, 15); + label3.TabIndex = 5; + label3.Text = "URL:"; + label3.Click += label3_Click; + // + // tbOrgId + // + tbOrgId.Location = new System.Drawing.Point(146, 194); + tbOrgId.Name = "tbOrgId"; + tbOrgId.Size = new System.Drawing.Size(252, 23); + tbOrgId.TabIndex = 8; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(47, 197); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(92, 15); + label4.TabIndex = 7; + label4.Text = "Organisation ID:"; + // + // cbAccessCredentials + // + cbAccessCredentials.FormattingEnabled = true; + cbAccessCredentials.Location = new System.Drawing.Point(146, 236); + cbAccessCredentials.Name = "cbAccessCredentials"; + cbAccessCredentials.Size = new System.Drawing.Size(252, 23); + cbAccessCredentials.TabIndex = 9; + // + // label5 + // + label5.AutoSize = true; + label5.Location = new System.Drawing.Point(1, 239); + label5.Name = "label5"; + label5.Size = new System.Drawing.Size(135, 15); + label5.TabIndex = 10; + label5.Text = "Data Access Credentials:"; + // + // DatasetProviderConfigurationUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + Controls.Add(label5); + Controls.Add(cbAccessCredentials); + Controls.Add(tbOrgId); + Controls.Add(label4); + Controls.Add(tbUrl); + Controls.Add(label3); + Controls.Add(tbType); + Controls.Add(label2); + Controls.Add(btnSave); + Controls.Add(tbName); + Controls.Add(label1); + Name = "DatasetProviderConfigurationUI"; + Size = new System.Drawing.Size(1003, 530); + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private System.Windows.Forms.TextBox tbName; + private System.Windows.Forms.Button btnSave; + private System.Windows.Forms.TextBox tbType; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.TextBox tbOrgId; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.ComboBox cbAccessCredentials; + private System.Windows.Forms.Label label5; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs new file mode 100644 index 0000000000..c2c24a99dd --- /dev/null +++ b/Rdmp.UI/SubComponents/DatasetProviderConfigurationUI.cs @@ -0,0 +1,73 @@ +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.Refreshing; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SubComponents; + +public partial class DatasetProviderConfigurationUI : DatasetProviderConfigurationUI_Design, IRefreshBusSubscriber +{ + private DatasetProviderConfiguration _configuration; + private IActivateItems _activator; + public DatasetProviderConfigurationUI() + { + InitializeComponent(); + } + + public override void SetDatabaseObject(IActivateItems activator, DatasetProviderConfiguration databaseObject) + { + _activator = activator; + base.SetDatabaseObject(activator, databaseObject); + _configuration = databaseObject; + tbName.Text = _configuration.Name; + tbOrgId.Text = _configuration.Organisation_ID; + tbType.Text = _configuration.Type; + tbUrl.Text = _configuration.Url; + cbAccessCredentials.Items.Clear(); + var credentials = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToArray(); + cbAccessCredentials.Items.AddRange(credentials); + cbAccessCredentials.SelectedIndex = Array.FindIndex(credentials, c => c.ID == _configuration.DataAccessCredentials_ID); + } + + private void label1_Click(object sender, EventArgs e) + { + + } + + private void label3_Click(object sender, EventArgs e) + { + + } + + private void btnSave_Click(object sender, EventArgs e) + { + _configuration.Name = tbName.Text; + _configuration.Url = tbUrl.Text; + _configuration.DataAccessCredentials_ID = ((DataAccessCredentials)cbAccessCredentials.SelectedItem).ID; + _configuration.Organisation_ID = tbOrgId.Text; + _configuration.SaveToDatabase(); + Publish(_configuration); + _activator.Show("Updated Dataset Provider Configuration"); + } + + public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) + { + } +} + +[TypeDescriptionProvider( + typeof(AbstractControlDescriptionProvider))] +public abstract class + DatasetProviderConfigurationUI_Design : RDMPSingleDatabaseObjectControl +{ +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs b/Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs new file mode 100644 index 0000000000..14cbf2ae48 --- /dev/null +++ b/Rdmp.UI/SubComponents/LinkDatasetDialog.Designer.cs @@ -0,0 +1,108 @@ +namespace Rdmp.UI.SubComponents +{ + partial class LinkDatasetDialog + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + groupBox2 = new System.Windows.Forms.GroupBox(); + tbID = new System.Windows.Forms.TextBox(); + groupBox3 = new System.Windows.Forms.GroupBox(); + cbPovider = new System.Windows.Forms.ComboBox(); + button1 = new System.Windows.Forms.Button(); + groupBox2.SuspendLayout(); + groupBox3.SuspendLayout(); + SuspendLayout(); + // + // groupBox2 + // + groupBox2.Controls.Add(tbID); + groupBox2.Location = new System.Drawing.Point(12, 82); + groupBox2.Name = "groupBox2"; + groupBox2.Size = new System.Drawing.Size(200, 60); + groupBox2.TabIndex = 2; + groupBox2.TabStop = false; + groupBox2.Text = "Dataset ID"; + // + // tbID + // + tbID.Location = new System.Drawing.Point(6, 20); + tbID.Name = "tbID"; + tbID.Size = new System.Drawing.Size(188, 23); + tbID.TabIndex = 0; + // + // groupBox3 + // + groupBox3.Controls.Add(cbPovider); + groupBox3.Location = new System.Drawing.Point(12, 8); + groupBox3.Name = "groupBox3"; + groupBox3.Size = new System.Drawing.Size(200, 60); + groupBox3.TabIndex = 2; + groupBox3.TabStop = false; + groupBox3.Text = "Dataset Provider"; + // + // cbPovider + // + cbPovider.FormattingEnabled = true; + cbPovider.Location = new System.Drawing.Point(6, 22); + cbPovider.Name = "cbPovider"; + cbPovider.Size = new System.Drawing.Size(188, 23); + cbPovider.TabIndex = 1; + // + // button1 + // + button1.Location = new System.Drawing.Point(216, 155); + button1.Name = "button1"; + button1.Size = new System.Drawing.Size(89, 23); + button1.TabIndex = 3; + button1.Text = "Link Dataset"; + button1.UseVisualStyleBackColor = true; + button1.Click += button1_Click; + // + // LinkDatasetDialog + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(800, 450); + Controls.Add(button1); + Controls.Add(groupBox3); + Controls.Add(groupBox2); + Name = "LinkDatasetDialog"; + Text = "Link Existing Dataset"; + groupBox2.ResumeLayout(false); + groupBox2.PerformLayout(); + groupBox3.ResumeLayout(false); + ResumeLayout(false); + } + + #endregion + private System.Windows.Forms.GroupBox groupBox2; + private System.Windows.Forms.TextBox tbID; + private System.Windows.Forms.GroupBox groupBox3; + private System.Windows.Forms.ComboBox cbPovider; + private System.Windows.Forms.Button button1; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SubComponents/LinkDatasetDialog.cs b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs new file mode 100644 index 0000000000..bb33cf05dc --- /dev/null +++ b/Rdmp.UI/SubComponents/LinkDatasetDialog.cs @@ -0,0 +1,56 @@ +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data; +using Rdmp.UI.ItemActivation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; +using System.Security.Policy; + +namespace Rdmp.UI.SubComponents +{ + public partial class LinkDatasetDialog : Form + { + IActivateItems _activator; + private Catalogue _catalogue; + public LinkDatasetDialog() + { + InitializeComponent(); + } + + public void Setup(IActivateItems activator, Catalogue catalogue) + { + _activator = activator; + _catalogue = catalogue; + cbPovider.Items.AddRange(_activator.RepositoryLocator.CatalogueRepository.GetAllObjects()); + } + + private void button1_Click(object sender, EventArgs e) + { + var provider = ((DatasetProviderConfiguration)cbPovider.SelectedItem).GetProviderInstance(_activator); + Dataset dataset; + var fetchedDataset = provider.FetchDatasetByID(int.Parse(tbID.Text)); + if (_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.Url).Any()) + { + dataset = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Url", fetchedDataset.Url).First(); + } + else + { + dataset = provider.AddExistingDatasetWithReturn(null, tbID.Text); + } + if (!_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Dataset_ID", dataset.ID).Where(l => l.Catalogue.ID == _catalogue.ID).Any()) + { + var linkage = new CatalogueDatasetLinkage(_activator.RepositoryLocator.CatalogueRepository, _catalogue, dataset); + linkage.SaveToDatabase(); + } + _activator.Publish(_catalogue); + Close(); + + } + } +} diff --git a/Rdmp.UI/TestsAndSetup/ServicePropogation/RDMPSingleDatabaseObjectControl.cs b/Rdmp.UI/TestsAndSetup/ServicePropogation/RDMPSingleDatabaseObjectControl.cs index 9286ff82da..3e45fdc7be 100644 --- a/Rdmp.UI/TestsAndSetup/ServicePropogation/RDMPSingleDatabaseObjectControl.cs +++ b/Rdmp.UI/TestsAndSetup/ServicePropogation/RDMPSingleDatabaseObjectControl.cs @@ -118,7 +118,7 @@ public virtual void SetDatabaseObject(IActivateItems activator, T databaseObject if (databaseObject is IMightBeReadOnly ro) { - if (ro.ShouldBeReadOnly(out var reason)) + if (ro.ShouldBeReadOnly(this.GetType().Name, out var reason)) { _readonlyIndicator.Text = reason; Controls.Add(_readonlyIndicator); diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 559a9fe198..b3b136a899 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,7 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("9.0.0")] -[assembly: AssemblyFileVersion("9.0.0")] -[assembly: AssemblyInformationalVersion("9.0.0")] + +[assembly: AssemblyVersion("9.1.1")] +[assembly: AssemblyFileVersion("9.1.1")] +[assembly: AssemblyInformationalVersion("9.1.1")] diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index 9569870268..6e3b082d47 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -17,6 +17,7 @@ using FAnsi.Implementations.Oracle; using FAnsi.Implementations.PostgreSql; using NUnit.Framework; +using Org.BouncyCastle.Asn1.X509.Qualified; using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandLine.Interactive; using Rdmp.Core.Curation; @@ -27,6 +28,7 @@ using Rdmp.Core.Curation.Data.Cohort.Joinables; using Rdmp.Core.Curation.Data.Dashboarding; using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.Curation.Data.Datasets; using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Pipelines; @@ -604,12 +606,27 @@ public static T WhenIHaveA(MemoryDataExportRepository repository) where T : D { return (T)(object)new RegexRedactionKey(repository.CatalogueRepository,WhenIHaveA(repository),WhenIHaveA(repository),"PK"); } + if(typeof(T) == typeof(CatalogueDatasetLinkage)) + { + return (T)(object)new CatalogueDatasetLinkage(repository.CatalogueRepository, WhenIHaveA(repository), WhenIHaveA(repository)); + } + if(typeof(T) == typeof(Dataset)) + { + return (T)(object)new Dataset(repository.CatalogueRepository, "Dataset"); + } + if (typeof(T) == typeof(PluginDataset)) + { + return (T)(object)new PluginDataset(repository.CatalogueRepository, "Plugin Dataset"); + } + if (typeof(T) == typeof(DatasetProviderConfiguration)) + { + return (T)(object)new DatasetProviderConfiguration(repository.CatalogueRepository, "","","",WhenIHaveA(repository).ID,""); + } if(typeof(T) == typeof(ExtractableDataSetProject)) { return (T)(object)new ExtractableDataSetProject(repository, WhenIHaveA(repository), WhenIHaveA(repository)); } - throw new TestCaseNotWrittenYetException(typeof(T)); } @@ -828,4 +845,4 @@ private MethodInfo GetWhenIHaveAMethod() { return typeof(UnitTests).GetMethod(nameof(WhenIHaveA), 1, new[] { typeof(MemoryDataExportRepository) }); } -} \ No newline at end of file +} diff --git a/rdmp-client.xml b/rdmp-client.xml index 78c1ea3da3..a2832499f4 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -1,7 +1,7 @@ - 9.0.0.0 - https://github.com/HicServices/RDMP/releases/download/v9.0.0/rdmp-9.0.0-client.zip + 9.1.1.0 + https://github.com/HicServices/RDMP/releases/download/v9.1.0/rdmp-9.1.0-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true