Skip to content

[Datasets] Update use of cold storage & extractability wip #2183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
JFriel wants to merge 7 commits into
base: task/RDMP-33-dataset-integration-interface
Choose a base branch
from
Draft

[Datasets] Update use of cold storage & extractability wip #2183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a67e838
add internal switch
JFriel Apr 25, 2025
5b4dcba
update tests
JFriel Apr 25, 2025
93b15a3
fix tests
JFriel Apr 25, 2025
afba546
fix up tests
JFriel Apr 25, 2025
3172b92
Merge branch 'task/RDMP-33-dataset-integration-interface' of https://…
JFriel May 6, 2025
a9abc6a
tidy up and use correct images
JFriel May 6, 2025
e5a79cb
fix test
JFriel May 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -636,7 +636,6 @@ public override void Wait(string title, Task task, CancellationTokenSource cts)
public override IEnumerable<Type> GetIgnoredCommands()
{
yield return typeof(ExecuteCommandRefreshObject);
yield return typeof(ExecuteCommandChangeExtractability);
yield return typeof(ExecuteCommandOpenInExplorer);
yield return typeof(ExecuteCommandDeletePlugin);
yield return typeof(ExecuteCommandCreateNewFileBasedProcessTask);
Expand Down
1 change: 0 additions & 1 deletion Rdmp.Core.Tests/CommandExecution/TestCommandsAreSupported.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ public void Init()
[TestCase(typeof(ExecuteCommandAssociateCatalogueWithLoadMetadata))]
[TestCase(typeof(ExecuteCommandAssociateCohortIdentificationConfigurationWithProject))]
[TestCase(typeof(ExecuteCommandBulkImportTableInfos))]
[TestCase(typeof(ExecuteCommandChangeExtractability))]
[TestCase(typeof(ExecuteCommandChangeExtractionCategory))]
[TestCase(typeof(ExecuteCommandChangeLoadStage))]
[TestCase(typeof(ExecuteCommandCheck))]
Expand Down
10 changes: 3 additions & 7 deletions ...ation/ObscureDependencyTests/BetweenCatalogueAndDataExportObscureDependencyFinderTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,10 @@ public void PreventDeletingCatalogueBecauseOfLinkedDatasetTest()
var dataset = new ExtractableDataSet(DataExportRepository, cata);

//and suddenly we cannot delete the catalogue
var ex = Assert.Throws<Exception>(() => obscura.ThrowIfDeleteDisallowed(cata));
Assert.That(ex.Message, Does.Contain("Cannot delete Catalogue MyCata because there are ExtractableDataSets which depend on them "));
Assert.DoesNotThrow(() => obscura.ThrowIfDeleteDisallowed(cata));

//also if we try to force through a delete it should behave in identical manner
var ex2 = Assert.Throws<Exception>(cata.DeleteInDatabase);
Assert.That(ex2.Message, Does.Contain("Cannot delete Catalogue MyCata because there are ExtractableDataSets which depend on them "));
Assert.DoesNotThrow(cata.DeleteInDatabase);

//now we delete the linked dataset
dataset.DeleteInDatabase();
Expand Down Expand Up @@ -65,9 +63,7 @@ public void AllowDeletingWhenDataExportManagerIsNotSet()
var cata = new Catalogue(CatalogueRepository, "MyCata");
var dataset = new ExtractableDataSet(DataExportRepository, cata);

//we cannot delete it because there is a dependency
var ex = Assert.Throws<Exception>(() => obscura1.ThrowIfDeleteDisallowed(cata));
Assert.That(ex.Message, Does.Contain("Cannot delete Catalogue MyCata because there are ExtractableDataSets which depend on them "));
Assert.DoesNotThrow(() => obscura1.ThrowIfDeleteDisallowed(cata));

//the second finder simulates when the repository locator doesn't have a record of the data export repository so it is unable to check it so it will let you delete it just fine
Assert.DoesNotThrow(() => obscura2.ThrowIfDeleteDisallowed(cata));
Expand Down
20 changes: 15 additions & 5 deletions ...aExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,9 @@ public void ReExtractToADatabaseWithNoNewData()
Name = "ext1",
Cohort_ID = extractableCohort.ID
};
ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue));
var existingExtractableDataSet = DataExportRepository.GetAllObjectsWhere<ExtractableDataSet>("Catalogue_ID", catalogue.ID).FirstOrDefault();

ec.AddDatasetToConfiguration( existingExtractableDataSet ?? new ExtractableDataSet(DataExportRepository, catalogue));

ec.SaveToDatabase();

Expand Down Expand Up @@ -409,7 +411,9 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs()
Name = "ext1",
Cohort_ID = extractableCohort.ID
};
ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue));
var existingExtractableDataSet = DataExportRepository.GetAllObjectsWhere<ExtractableDataSet>("Catalogue_ID", catalogue.ID).FirstOrDefault();

ec.AddDatasetToConfiguration( existingExtractableDataSet ?? new ExtractableDataSet(DataExportRepository, catalogue));

ec.SaveToDatabase();

Expand Down Expand Up @@ -671,7 +675,9 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK()
Name = "ext1",
Cohort_ID = extractableCohort.ID
};
ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue));
var existingExtractableDataSet = DataExportRepository.GetAllObjectsWhere<ExtractableDataSet>("Catalogue_ID", catalogue.ID).FirstOrDefault();

ec.AddDatasetToConfiguration(existingExtractableDataSet ?? new ExtractableDataSet(DataExportRepository, catalogue));

ec.SaveToDatabase();

Expand Down Expand Up @@ -937,7 +943,9 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK()
Name = "ext1",
Cohort_ID = extractableCohort.ID
};
ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue));
var existingExtractableDataSet = DataExportRepository.GetAllObjectsWhere<ExtractableDataSet>("Catalogue_ID", catalogue.ID).FirstOrDefault();

ec.AddDatasetToConfiguration(existingExtractableDataSet??new ExtractableDataSet(DataExportRepository, catalogue));

ec.SaveToDatabase();

Expand Down Expand Up @@ -1196,7 +1204,9 @@ public void ExtractToDatabaseUseTriggers()
Name = "ext1",
Cohort_ID = extractableCohort.ID
};
ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue));
var existingExtractableDataSet = DataExportRepository.GetAllObjectsWhere<ExtractableDataSet>("Catalogue_ID", catalogue.ID).FirstOrDefault();

ec.AddDatasetToConfiguration(existingExtractableDataSet??new ExtractableDataSet(DataExportRepository, catalogue));

ec.SaveToDatabase();

Expand Down
15 changes: 10 additions & 5 deletions Rdmp.Core/CommandExecution/AtomicCommandFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,11 +131,16 @@ public IEnumerable<IAtomicCommand> CreateCommands(object o)
};
if (!isApiCall)
{
yield return new ExecuteCommandChangeExtractability(_activator, c)
{
Weight = -99.0010f,
SuggestedCategory = Extraction
};
yield return c.IsInternalDataset ?
new ExecuteCommandMakeCatalogueNotInternal(_activator, c)
{
Weight = -99.0008f,
SuggestedCategory = Extraction
} : new ExecuteCommandMakeCatalogueInternal(_activator, c)
{
Weight = -99.0008f,
SuggestedCategory = Extraction
};

yield return c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)
? new ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(_activator, c)
Expand Down
88 changes: 0 additions & 88 deletions Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractability.cs
View file
Open in desktop

This file was deleted.

81 changes: 81 additions & 0 deletions Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.Icons.IconProvision;
using Rdmp.Core.Repositories.Construction;
using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;

namespace Rdmp.Core.CommandExecution.AtomicCommands;

public class ExecuteCommandMakeCatalogueInternal : BasicCommandExecution, IAtomicCommandWithTarget
{
private ICatalogue _catalogue;

[UseWithObjectConstructor]
public ExecuteCommandMakeCatalogueInternal(IBasicActivateItems itemActivator, ICatalogue catalogue) : this(itemActivator)
{
SetCatalogue(catalogue);
}

public ExecuteCommandMakeCatalogueInternal(IBasicActivateItems itemActivator) : base(itemActivator)
{
UseTripleDotSuffix = true;
}

public override string GetCommandHelp() =>
"Mark the Catalogue as Internal to restrict its extractability";

public override void Execute()
{
if (_catalogue == null)
SetCatalogue(SelectOne<Catalogue>(BasicActivator.RepositoryLocator.CatalogueRepository));

if (_catalogue == null)
return;

_catalogue.IsInternalDataset = true;
_catalogue.SaveToDatabase();
Publish(_catalogue);
}

public override Image<Rgba32> GetImage(IIconProvider iconProvider) => BasicActivator.CoreIconProvider.GetImage(_catalogue, OverlayKind.Extractable_Internal);

public IAtomicCommandWithTarget SetTarget(DatabaseEntity target)
{
switch (target)
{
case Catalogue catalogue:
SetCatalogue(catalogue);
break;
default:
break;
}

return this;
}

private void SetCatalogue(ICatalogue catalogue)
{
ResetImpossibleness();

_catalogue = catalogue;

if (catalogue == null)
{
SetImpossible("Catalogue cannot be null");
return;
}

if (_catalogue.IsInternalDataset)
SetImpossible("Catalogue is already marked as Internal");
}
}
82 changes: 82 additions & 0 deletions Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueNotInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.Icons.IconProvision;
using Rdmp.Core.Repositories.Construction;
using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;

namespace Rdmp.Core.CommandExecution.AtomicCommands;

public class ExecuteCommandMakeCatalogueNotInternal : BasicCommandExecution, IAtomicCommandWithTarget
{
private ICatalogue _catalogue;

[UseWithObjectConstructor]
public ExecuteCommandMakeCatalogueNotInternal(IBasicActivateItems itemActivator, ICatalogue catalogue) : this(itemActivator)
{
SetCatalogue(catalogue);
}

public ExecuteCommandMakeCatalogueNotInternal(IBasicActivateItems itemActivator) : base(itemActivator)
{
UseTripleDotSuffix = true;
}

public override string GetCommandHelp() =>
"Mark the Catalogue as not Internal";

public override void Execute()
{
if (_catalogue == null)
SetCatalogue(SelectOne<Catalogue>(BasicActivator.RepositoryLocator.CatalogueRepository));

if (_catalogue == null)
return;

_catalogue.IsInternalDataset = false;
_catalogue.SaveToDatabase();
Publish(_catalogue);
}

public override Image<Rgba32> GetImage(IIconProvider iconProvider) =>
Image.Load<Rgba32>(CatalogueIcons.Catalogue);

public IAtomicCommandWithTarget SetTarget(DatabaseEntity target)
{
switch (target)
{
case Catalogue catalogue:
SetCatalogue(catalogue);
break;
default:
break;
}

return this;
}

private void SetCatalogue(ICatalogue catalogue)
{
ResetImpossibleness();

_catalogue = catalogue;

if (catalogue == null)
{
SetImpossible("Catalogue cannot be null");
return;
}

if (!_catalogue.IsInternalDataset)
SetImpossible("Catalogue is not marked as Internal");
}
}
3 changes: 0 additions & 3 deletions Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandMakeCatalogueProjectSpecific.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,6 @@ private void SetCatalogue(ICatalogue catalogue)
if (status.IsProjectSpecific)
SetImpossible("Catalogue is already Project Specific");

if (!status.IsExtractable)
SetImpossible("Catalogue must first be made Extractable");

var ei = _catalogue.GetAllExtractionInformation(ExtractionCategory.Any);
if (!ei.Any())
SetImpossible("Catalogue has no extractable columns");
Expand Down
6 changes: 0 additions & 6 deletions .../CommandExecution/AtomicCommands/ExecuteCommandMakeProjectSpecificCatalogueNormalAgain.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,6 @@ public ExecuteCommandMakeProjectSpecificCatalogueNormalAgain(IBasicActivateItems
_extractableDataSet = dataExportRepository.GetAllObjectsWithParent<ExtractableDataSet>(catalogue)
.SingleOrDefault();

if (_extractableDataSet == null)
{
SetImpossible("Catalogue is not extractable");
return;
}

if (_extractableDataSet.Project_ID == null)
{
SetImpossible("Catalogue is not a project specific Catalogue");
Expand Down
Loading
Loading