From 6348de52ca8812a7ba6d6a602ff5fa756f977205 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 29 Apr 2025 09:05:24 +0100 Subject: [PATCH 01/20] add start of HSR integration --- .../Curation/Data/Datasets/HDR/HDRDataset.cs | 31 ++ .../Datasets/HDR/HDRDatasetItems/Access.cs | 23 ++ .../HDR/HDRDatasetItems/Accessibility.cs | 18 ++ .../Datasets/HDR/HDRDatasetItems/Coverage.cs | 22 ++ .../Data/Datasets/HDR/HDRDatasetItems/Data.cs | 54 ++++ .../HDR/HDRDatasetItems/DataCustodian.cs | 21 ++ .../HDR/HDRDatasetItems/Documentation.cs | 18 ++ .../HDRDatasetItems/EnrichmentAndLinkage.cs | 32 ++ .../HDR/HDRDatasetItems/FormatAndStandards.cs | 19 ++ .../Datasets/HDR/HDRDatasetItems/Metadata.cs | 38 +++ .../HDR/HDRDatasetItems/NamedEntity.cs | 21 ++ .../Datasets/HDR/HDRDatasetItems/Origin.cs | 21 ++ .../HDR/HDRDatasetItems/Provenance.cs | 17 ++ .../Datasets/HDR/HDRDatasetItems/Revision.cs | 17 ++ .../HDR/HDRDatasetItems/SpatialCoverage.cs | 21 ++ .../HDR/HDRDatasetItems/StructuralMetadata.cs | 19 ++ .../Datasets/HDR/HDRDatasetItems/Summary.cs | 23 ++ .../Data/Datasets/HDR/HDRDatasetItems/Team.cs | 47 +++ .../Datasets/HDR/HDRDatasetItems/Temporal.cs | 26 ++ .../Datasets/HDR/HDRDatasetItems/Usage.cs | 18 ++ .../Datasets/HDR/HDRDatasetItems/Version.cs | 26 ++ .../Data/Datasets/HDR/HDRDatasetPatch.cs | 177 +++++++++++ .../Data/Datasets/HDR/HDRDatasetProvider.cs | 284 ++++++++++++++++++ .../HDR/Helpers/CustomDateTimeConverter.cs | 27 ++ ...ustomDateTimeConverterThreeMilliseconds.cs | 27 ++ .../Collections/ConfigurationsCollectionUI.cs | 9 +- ...ecuteCommandCreateNewHDRConfigurationUI.cs | 29 ++ ...xecuteCommandImportExistingHDRDatasetUI.cs | 28 ++ .../CreateNewHDRConfigurationUI.Designer.cs | 250 +++++++++++++++ .../HDR/CreateNewHDRConfigurationUI.cs | 79 +++++ .../ImportExistingHDRDatasetUI.Designer.cs | 134 +++++++++ .../HDR/ImportExistingHDRDatasetUI.cs | 76 +++++ Tests.Common/UnitTests.cs | 5 + 33 files changed, 1656 insertions(+), 1 deletion(-) create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Access.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Accessibility.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Coverage.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Data.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/DataCustodian.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Documentation.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/EnrichmentAndLinkage.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/FormatAndStandards.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Metadata.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/NamedEntity.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Origin.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Provenance.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Revision.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/SpatialCoverage.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/StructuralMetadata.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Summary.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Team.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Temporal.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Usage.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Version.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/Helpers/CustomDateTimeConverter.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/Helpers/CustomDateTimeConverterThreeMilliseconds.cs create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewHDRConfigurationUI.cs create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingHDRDatasetUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.Designer.cs create mode 100644 Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.cs diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs new file mode 100644 index 0000000000..970be25ec6 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs @@ -0,0 +1,31 @@ +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR +{ + /// + /// + /// + public class HDRDataset : PluginDataset + { + public HDRDataset() : base() { } + public HDRDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name) + { + } + + public HDRDatasetItems.Data data { get; set; } + + public override string GetID() + { + return data.id.ToString(); + } + public override string GetRemoteID() + { + return Url.Split('?')[0].Split('/').Last(); + } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Access.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Access.cs new file mode 100644 index 0000000000..a8cf4fa7fb --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Access.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Access + { + public string deliveryLeadTime { get; set; } + public List jurisdiction { get; set; } + public string dataController { get; set; } + public string dataProcessor { get; set; } + public string accessRights { get; set; } + public string accessService { get; set; } + public string accessRequestCost { get; set; } + public object accessServiceCategory { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Accessibility.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Accessibility.cs new file mode 100644 index 0000000000..6e657152c7 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Accessibility.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Accessibility + { + public Access access { get; set; } + public Usage usage { get; set; } + public FormatAndStandards formatAndStandards { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Coverage.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Coverage.cs new file mode 100644 index 0000000000..ef1678c629 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Coverage.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Coverage + { + public object pathway { get; set; } + public string spatial { get; set; } + public object followUp { get; set; } + public object datasetCompleteness { get; set; } + public List materialType { get; set; } + public int typicalAgeRangeMin { get; set; } + public int typicalAgeRangeMax { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Data.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Data.cs new file mode 100644 index 0000000000..fb482dd0df --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Data.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using static NPOI.HSSF.Util.HSSFColor; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Data + { + public int id { get; set; } + public object mongo_object_id { get; set; } + public object mongo_id { get; set; } + public object mongo_pid { get; set; } + public object datasetid { get; set; } + public string pid { get; set; } + public object source { get; set; } + public int discourse_topic_id { get; set; } + public bool is_cohort_discovery { get; set; } + public int commercial_use { get; set; } + public int state_id { get; set; } + public int uploader_id { get; set; } + public int metadataquality_id { get; set; } + public int user_id { get; set; } + public int team_id { get; set; } + public int views_count { get; set; } + public int views_prev_count { get; set; } + public int has_technical_details { get; set; } + public string created { get; set; } + public string updated { get; set; } + public string submitted { get; set; } + public object published { get; set; } + public DateTime created_at { get; set; } + public DateTime updated_at { get; set; } + public object deleted_at { get; set; } + public string create_origin { get; set; } + public string status { get; set; } + public int durs_count { get; set; } + public int publications_count { get; set; } + public int tools_count { get; set; } + public int collections_count { get; set; } + public List spatialCoverage { get; set; } + public List durs { get; set; } + public List publications { get; set; } + public List named_entities { get; set; } + public List collections { get; set; } + public List versions { get; set; } + public Team team { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/DataCustodian.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/DataCustodian.cs new file mode 100644 index 0000000000..b229b14ed1 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/DataCustodian.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class DataCustodian + { + public string name { get; set; } + public string identifier { get; set; } + public string contactPoint { get; set; } + public object logo { get; set; } + public object description { get; set; } + public object memberOf { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Documentation.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Documentation.cs new file mode 100644 index 0000000000..177edbdd41 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Documentation.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Documentation + { + public string description { get; set; } + public object associatedMedia { get; set; } + public string inPipeline { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/EnrichmentAndLinkage.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/EnrichmentAndLinkage.cs new file mode 100644 index 0000000000..69583497da --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/EnrichmentAndLinkage.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class EnrichmentAndLinkage + { + public List tools { get; set; } + public List investigations { get; set; } + public List publicationAboutDataset { get; set; } + public List publicationUsingDataset { get; set; } + + [JsonIgnore] + public object derivedFrom { get; set; } + + [JsonIgnore] + public object isPartOf { get; set; } + + [JsonIgnore] + public object linkableDatasets { get; set; } + + [JsonIgnore] + public object similarToDatasets { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/FormatAndStandards.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/FormatAndStandards.cs new file mode 100644 index 0000000000..334ec7af94 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/FormatAndStandards.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class FormatAndStandards + { + public List conformsTo { get; set; } + public List vocabularyEncodingScheme { get; set; } + public List language { get; set; } + public List format { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Metadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Metadata.cs new file mode 100644 index 0000000000..d17e508565 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Metadata.cs @@ -0,0 +1,38 @@ +using MathNet.Numerics; +using Rdmp.Core.Curation.Data.Datasets.HDR.Helpers; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Metadata + { + public Metadata metadata { get; set; } + public string identifier { get; set; } + + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + public DateTime issued { get; set; } + + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + public DateTime modified { get; set; } + public List revisions { get; set; } + public string version { get; set; } + public Summary summary { get; set; } + public Documentation documentation { get; set; } + public Coverage coverage { get; set; } + public Provenance provenance { get; set; } + public Accessibility accessibility { get; set; } + public EnrichmentAndLinkage enrichmentAndLinkage { get; set; } + public List observations { get; set; } + public StructuralMetadata structuralMetadata { get; set; } + public object demographicFrequency { get; set; } + public object omics { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/NamedEntity.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/NamedEntity.cs new file mode 100644 index 0000000000..4674d546a3 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/NamedEntity.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class NamedEntity + { + public int id { get; set; } + public string name { get; set; } + public DateTime created_at { get; set; } + public DateTime updated_at { get; set; } + public object deleted_at { get; set; } + public List dataset_version_ids { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Origin.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Origin.cs new file mode 100644 index 0000000000..c281dcd93a --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Origin.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Origin + { + public List purpose { get; set; } + public List source { get; set; } + public List collectionSource { get; set; } + public List datasetType { get; set; } + public List datasetSubType { get; set; } + public string imageContrast { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Provenance.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Provenance.cs new file mode 100644 index 0000000000..a72b4a4cb9 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Provenance.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Provenance + { + public Origin origin { get; set; } + public Temporal temporal { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Revision.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Revision.cs new file mode 100644 index 0000000000..22236ed4bd --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Revision.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Revision + { + public string version { get; set; } + public string url { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/SpatialCoverage.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/SpatialCoverage.cs new file mode 100644 index 0000000000..635b0f9ee8 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/SpatialCoverage.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class SpatialCoverage + { + public int id { get; set; } + public DateTime created_at { get; set; } + public DateTime updated_at { get; set; } + public string region { get; set; } + public bool enabled { get; set; } + public List dataset_version_ids { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/StructuralMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/StructuralMetadata.cs new file mode 100644 index 0000000000..17cfaa55fc --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/StructuralMetadata.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class StructuralMetadata + { + public List tables { get; set; } + [JsonIgnore] + public object syntheticDataWebLink { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Summary.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Summary.cs new file mode 100644 index 0000000000..fdb64724c1 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Summary.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Summary + { + public string @abstract { get; set; } + public string contactPoint { get; set; } + public List keywords { get; set; } + public object doiName { get; set; } + public string title { get; set; } + public DataCustodian dataCustodian { get; set; } + public int populationSize { get; set; } + public object alternateIdentifiers { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Team.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Team.cs new file mode 100644 index 0000000000..9837ac6d29 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Team.cs @@ -0,0 +1,47 @@ +using Rdmp.Core.Curation.Data.Datasets.HDR.Helpers; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Team + { + public int id { get; set; } + public string pid { get; set; } + + [JsonConverter(typeof(CustomDateTimeConverter))] + public DateTime created_at { get; set; } + public DateTime updated_at { get; set; } + public object deleted_at { get; set; } + public string name { get; set; } + public bool enabled { get; set; } + public bool allows_messaging { get; set; } + public bool workflow_enabled { get; set; } + public bool access_requests_management { get; set; } + public bool uses_5_safes { get; set; } + public bool is_admin { get; set; } + public string team_logo { get; set; } + public string member_of { get; set; } + public object contact_point { get; set; } + public string application_form_updated_by { get; set; } + public string application_form_updated_on { get; set; } + public string mongo_object_id { get; set; } + public bool notification_status { get; set; } + public bool is_question_bank { get; set; } + public bool is_provider { get; set; } + public object url { get; set; } + public object introduction { get; set; } + public object dar_modal_header { get; set; } + public object dar_modal_content { get; set; } + public object dar_modal_footer { get; set; } + public bool is_dar { get; set; } + public object service { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Temporal.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Temporal.cs new file mode 100644 index 0000000000..d0718ce672 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Temporal.cs @@ -0,0 +1,26 @@ +using Rdmp.Core.Curation.Data.Datasets.HDR.Helpers; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Temporal + { + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + + public DateTime? endDate { get; set; } + + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + public DateTime startDate { get; set; } + public string timeLag { get; set; } + public string publishingFrequency { get; set; } + public object distributionReleaseDate { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Usage.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Usage.cs new file mode 100644 index 0000000000..0739c79082 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Usage.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Usage + { + public List dataUseLimitation { get; set; } + public object resourceCreator { get; set; } + public List dataUseRequirements { get; set; } = []; + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Version.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Version.cs new file mode 100644 index 0000000000..0d00627d2a --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Version.cs @@ -0,0 +1,26 @@ +using Org.BouncyCastle.Asn1.Cms; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + /// + /// + /// + public class Version + { + public int id { get; set; } + public DateTime created_at { get; set; } + public DateTime updated_at { get; set; } + public object deleted_at { get; set; } + public int dataset_id { get; set; } + public Metadata metadata { get; set; } + public int version { get; set; } + public object provider_team_id { get; set; } + public object application_type { get; set; } + public List reduced_linked_dataset_versions { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs new file mode 100644 index 0000000000..3fc3cb45e1 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs @@ -0,0 +1,177 @@ +using Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems; +using Rdmp.Core.Curation.Data.Datasets.HDR.Helpers; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR +{ + public class PatchMetadata + { + + public string schemaModel { get; set; } + public string schemaVersion { get; set; } + public PatchSubMetadata metadata { get; set; } + + + public PatchMetadata() { } + public PatchMetadata(Metadata existingMetadata) + { + + metadata = existingMetadata.metadata != null ? new PatchSubMetadata(existingMetadata.metadata) : null; + schemaModel = "HDRUK"; + schemaVersion = "3.0.0"; + } + } + + public class PatchSubMetadata + { + public object observations { get; set; } + public Coverage coverage { get; set; } + public object structuralMetadata { get; set; } + public object enrichmentAndLinkage { get; set; } + public Accessibility accessibility { get; set; } + + public string identifier { get; set; } + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + + public DateTime issued { get; set; } + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + + public DateTime modified { get; set; } + + public Provenance provenance { get; set; } + public object documentation { get; set; } + public Summary summary { get; set; } + public List revisions { get; set; } + + public string version { get; set; } + + public PatchSubMetadata(Metadata existingMetadata) + { + accessibility = existingMetadata.accessibility; + observations = existingMetadata.observations; + coverage = existingMetadata.coverage; + structuralMetadata = existingMetadata.structuralMetadata; + enrichmentAndLinkage = existingMetadata.enrichmentAndLinkage; + provenance = existingMetadata.provenance; + documentation = existingMetadata.documentation; + summary = existingMetadata.summary; + identifier = existingMetadata.identifier; + issued = existingMetadata.issued; + modified = existingMetadata.modified; + revisions = existingMetadata.revisions; + version = existingMetadata.version; + } + } + + public class HDRDatasetPatch + { + public int id { get; set; } + public object mongo_object_id { get; set; } + public object mongo_id { get; set; } + public object mongo_pid { get; set; } + public object datasetid { get; set; } + public string pid { get; set; } + public object source { get; set; } + public int discourse_topic_id { get; set; } + public bool is_cohort_discovery { get; set; } + public int commercial_use { get; set; } + public int state_id { get; set; } + public int uploader_id { get; set; } + public int metadataquality_id { get; set; } + public int user_id { get; set; } + public int team_id { get; set; } + public int views_count { get; set; } + public int views_prev_count { get; set; } + public int has_technical_details { get; set; } + public string created { get; set; } + public string updated { get; set; } + public string submitted { get; set; } + public object published { get; set; } + public DateTime created_at { get; set; } + public DateTime updated_at { get; set; } + public object deleted_at { get; set; } + public string create_origin { get; set; } + public string status { get; set; } + public int durs_count { get; set; } + public int publications_count { get; set; } + public int tools_count { get; set; } + public int collections_count { get; set; } + public List spatialCoverage { get; set; } + public List durs { get; set; } + public List publications { get; set; } + public List named_entities { get; set; } + public List collections { get; set; } + public Team team { get; set; } + public PatchMetadata metadata { get; set; } + + + public HDRDatasetPatch() { } + public HDRDatasetPatch(HDRDataset existingDataset) + { + id = existingDataset.data.id; + mongo_object_id = existingDataset.data.mongo_object_id; + mongo_id = existingDataset.data.mongo_id; + mongo_pid = existingDataset.data.mongo_pid; + datasetid = existingDataset.data.datasetid; + pid = existingDataset.data.pid; + source = existingDataset.data.source; + discourse_topic_id = existingDataset.data.discourse_topic_id; + is_cohort_discovery = existingDataset.data.is_cohort_discovery; + commercial_use = existingDataset.data.commercial_use; + state_id = existingDataset.data.state_id; + uploader_id = existingDataset.data.uploader_id; + metadataquality_id = existingDataset.data.metadataquality_id; + user_id = existingDataset.data.user_id; + team_id = existingDataset.data.team_id; + views_count = existingDataset.data.views_count; + views_prev_count = existingDataset.data.views_prev_count; + has_technical_details = existingDataset.data.has_technical_details; + created = existingDataset.data.created; + updated = existingDataset.data.updated; + submitted = existingDataset.data.submitted; + published = existingDataset.data.published; + created_at = existingDataset.data.created_at; + updated_at = existingDataset.data.updated_at; + deleted_at = existingDataset.data.deleted_at; + create_origin = existingDataset.data.create_origin; + status = existingDataset.data.status; + durs_count = existingDataset.data.durs_count; + publications_count = existingDataset.data.publications_count; + tools_count = existingDataset.data.tools_count; + collections_count = existingDataset.data.collections_count; + spatialCoverage = existingDataset.data.spatialCoverage; + durs = existingDataset.data.durs; + publications = existingDataset.data.publications; + named_entities = existingDataset.data.named_entities; + collections = existingDataset.data.collections; + team = existingDataset.data.team; + //metadata = new PatchMetadata(existingDataset.data.metadata); + metadata = new PatchMetadata(existingDataset.data.versions.First().metadata); + //if(metadata.metadata.accessibility.access.jurisdiction is string) + // metadata.metadata.accessibility.access.jurisdiction = metadata.metadata.accessibility.access.jurisdiction.ToString().Split(";,;"); + + //if (metadata.metadata.accessibility.formatAndStandards.conformsTo is string) + // metadata.metadata.accessibility.formatAndStandards.conformsTo = metadata.metadata.accessibility.formatAndStandards.conformsTo.ToString().Split(";,;"); + //if (metadata.metadata.accessibility.formatAndStandards.formats is string) + // metadata.metadata.accessibility.formatAndStandards.formats = metadata.metadata.accessibility.formatAndStandards.formats.ToString().Split(";,;"); + //if (metadata.metadata.accessibility.formatAndStandards.languages is string) + // metadata.metadata.accessibility.formatAndStandards.languages = metadata.metadata.accessibility.formatAndStandards.languages.ToString().Split(";,;"); + //if (metadata.metadata.accessibility.formatAndStandards.vocabularyEncodingSchemes is string) + // metadata.metadata.accessibility.formatAndStandards.vocabularyEncodingSchemes = metadata.metadata.accessibility.formatAndStandards.vocabularyEncodingSchemes.ToString().Split(";,;"); + + //if (metadata.metadata.accessibility.usage.dataUseLimitation is string) + // metadata.metadata.accessibility.usage.dataUseLimitation = metadata.metadata.accessibility.usage.dataUseLimitation.ToString().Split(";,;"); + if (metadata.metadata.accessibility.usage.dataUseRequirements is null) + metadata.metadata.accessibility.usage.dataUseRequirements = []; + + metadata.metadata.coverage.materialType = new List() { "Other" }; + //metadata.metadata.summary.dataCustodian.description = "Test"; + //metadata.metadata.summary.dataCustodian.logo = "https://cdn.britannica.com/92/100692-050-5B69B59B/Mallard.jpg"; + } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs new file mode 100644 index 0000000000..799fed64d5 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -0,0 +1,284 @@ +using Org.BouncyCastle.Asn1.Cms; +using Rdmp.Core.CommandExecution; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Rdmp.Core.Curation.Data.Datasets.HDR.Helpers; +using Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR +{ + /// + /// Provider for connecting to the HDR Gateway + /// + public class HDRDatasetProvider : PluginDatasetProvider + { + private HttpClient _client; + private DatasetProviderConfiguration _configuration; + public HDRDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration) : base(activator, configuration) + { + _client = new HttpClient(); + var credentials = Repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); + var apiKey = credentials.GetDecryptedPassword(); + _client.DefaultRequestHeaders.Add("x-application-id", credentials.Username); + _client.DefaultRequestHeaders.Add("x-client-id", apiKey); + _configuration = configuration; + } + public override Dataset AddExistingDatasetWithReturn(string name, string url) + { + var response = Task.Run(async () => await _client.GetAsync(url)).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + HDRDataset hdrDataset = JsonConvert.DeserializeObject(detailsString); + var datasetName = string.IsNullOrWhiteSpace(name) ? hdrDataset?.data.versions?.First().metadata.metadata.summary.title : name; + var dataset = new Dataset(Repository, datasetName) + { + Url = url, + Type = ToString(), + Provider_ID = Configuration.ID, + //DigitalObjectIdentifier = hdrDataset.data.versions.First().metadata.metadata.summary.doiName.ToString(), + Folder = $"\\{Configuration.Name}", + }; + dataset.SaveToDatabase(); + Activator.Publish(dataset); + return dataset; + } + else + { + throw new Exception("Cannot access dataset at provided url"); + } + } + + public override void AddExistingDataset(string name, string url) + { + AddExistingDatasetWithReturn(name, url); + } + + public override Dataset Create(Catalogue catalogue) + { + var url = Configuration.Url + "/v1/datasets"; + var serializeOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + IncludeFields = true + }; + using var stream = new MemoryStream(); + var ds = new HDRDatasetPatch(); + ds.metadata = new PatchMetadata() + { + schemaModel = "HDRUK", + schemaVersion = "3.0.0", + metadata = new PatchSubMetadata(new Metadata() + { + summary = new Summary() + { + title = catalogue.Name + } + }) + }; + System.Text.Json.JsonSerializer.Serialize(stream, ds, serializeOptions); + var jsonString = Encoding.UTF8.GetString(stream.ToArray()); + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + var response = Task.Run(async () => await _client.PostAsync(url, httpContent)).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + + } + throw new Exception("q"); + + } + + public HDRDataset FetchHDRDataset(Dataset dataset) + { + var response = Task.Run(async () => await _client.GetAsync($"{dataset.Url}?schema_model=HDRUK&schema_version=3.0.0")).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + Console.WriteLine(detailsString); + HDRDataset hdrDataset = JsonConvert.DeserializeObject(detailsString); + return hdrDataset; + } + throw new Exception("Unable to fetch HDR dataset"); + } + + public override Dataset FetchDatasetByID(int id) + { + var url = Configuration.Url + "/v1/datasets/" + id; + var response = Task.Run(async () => await _client.GetAsync($"{url}?schema_model=HDRUK&schema_version=3.0.0")).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + Console.WriteLine(detailsString); + HDRDataset hdrDataset = JsonConvert.DeserializeObject(detailsString); + return hdrDataset; + } + throw new Exception("Unable to fetch HDR dataset"); + } + + public override List FetchDatasets() + { + throw new NotImplementedException(); + } + + public override void Update(string uuid, PluginDataset datasetUpdates) + { + var serializeOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }; + serializeOptions.Converters.Add(new CustomDateTimeConverter()); + serializeOptions.Converters.Add(new CustomDateTimeConverterThreeMilliseconds()); + + var options = new JsonWriterOptions + { + Indented = true + }; + + using var stream = new MemoryStream(); + var update = (HDRDataset)datasetUpdates; + var updateObj = new HDRUpdateObject() + { + metadata = new HDRDatasetPatch((HDRDataset)datasetUpdates).metadata.metadata + }; + System.Text.Json.JsonSerializer.Serialize(stream, updateObj.metadata, serializeOptions); + var jsonString = "{\"metadata\":{\"metadata\":" + Encoding.UTF8.GetString(stream.ToArray()) + "}}"; + //var uri = $"{Configuration.Url}/v1/integrations/datasets/{uuid}?schema_model=HDRUK&schema_version=3.0.0"; + var uri = $"{Configuration.Url}/v1/integrations/datasets/{uuid}?input_schema=HDRUK&input_version=3.0.0"; + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + + var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; + if (response.StatusCode != HttpStatusCode.OK) + { + var x = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + throw new Exception(x); + } + } + + private string MapDataTypeToHDRDataType(string dt) + { + switch (dt) + { + case "HealthcareAndDisease": + return "Healthcare and disease"; + case "TreatmentsAndInterventions": + return "Treatments/Interventions"; + case "MeasurementsAndTests": + return "Measurements/Tests"; + case "ImagingTypes": + return "Imaging types"; + case "ImagingAreaOfTheBody": + return "Imaging area of the body"; + case "Omics": + return "Omics"; + case "Socioeconomic": + return "Socioeconomic"; + case "Lifestyle": + return "Lifestyle"; + case "Registry": + return "Registry"; + case "EnvironmentalAndEnergy": + return "Environmental and energy"; + case "InformationAndCommunication": + return "Information and communication"; + case "Politics": + return "Politics"; + default: + return dt; + } + } + + private string AddSpacesToSentence(string text, bool preserveAcronyms) + { + if (string.IsNullOrWhiteSpace(text)) + return string.Empty; + StringBuilder newText = new StringBuilder(text.Length * 2); + newText.Append(text[0]); + for (int i = 1; i < text.Length; i++) + { + if (char.IsUpper(text[i]) && (text[i - 1] != ' ' && !char.IsUpper(text[i - 1]) || + preserveAcronyms && char.IsUpper(text[i - 1]) && + i < text.Length - 1 && !char.IsUpper(text[i + 1]))) + newText.Append(' '); + newText.Append(text[i]); + } + return newText.ToString(); + } + + private string MapDataSubTypeToHDR(string dst) + { + if (dst == "ResearchDiseaseRegistry") return "Disease registry (research)"; + return AddSpacesToSentence(dst, true); + } + + private string MapTimeLagToHDR(string tl) + { + switch (tl) + { + case "LessThanAWeek": + return "Less than 1 week"; + case "OneToTwoWeeks": + return "1-2 weeks"; + case "TwoToFourWeeks": + return "2-4 weeks"; + case "OneToTwoMonths": + return "1-2 months"; + case "TwoToSixMonths": + return "2-6 months"; + case "SixMonthsPlus": + return "More than 6 months"; + case "NotApplicable": + return "Not applicable"; + default: + return tl; + } + } + + public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) + { + var hdrDataset = (HDRDataset)dataset; + hdrDataset.data.versions.First().metadata.metadata.summary.title = catalogue.Name; + hdrDataset.data.versions.First().metadata.metadata.summary.@abstract = catalogue.ShortDescription; + hdrDataset.data.versions.First().metadata.metadata.summary.contactPoint = catalogue.Administrative_contact_email; + hdrDataset.data.versions.First().metadata.metadata.summary.keywords = catalogue.Search_keywords.Split(',').Cast().ToList(); + hdrDataset.data.versions.First().metadata.metadata.summary.doiName = catalogue.Doi; + + hdrDataset.data.versions.First().metadata.metadata.documentation.description = catalogue.Description; + hdrDataset.data.versions.First().metadata.metadata.documentation.associatedMedia = catalogue.AssociatedMedia; + + hdrDataset.data.versions.First().metadata.metadata.coverage.spatial = catalogue.Geographical_coverage; + + hdrDataset.data.versions.First().metadata.metadata.provenance.origin.datasetType = catalogue.DataType.Split(",").Select(MapDataTypeToHDRDataType).ToList(); + hdrDataset.data.versions.First().metadata.metadata.provenance.origin.datasetSubType = catalogue.DataSubType.Split(",").Select(MapDataSubTypeToHDR).ToList(); + hdrDataset.data.versions.First().metadata.metadata.provenance.temporal.endDate = catalogue.EndDate.Value; + hdrDataset.data.versions.First().metadata.metadata.provenance.temporal.startDate = catalogue.StartDate.Value; + hdrDataset.data.versions.First().metadata.metadata.provenance.temporal.timeLag = MapTimeLagToHDR(catalogue.UpdateLag.ToString()); + hdrDataset.data.versions.First().metadata.metadata.provenance.temporal.publishingFrequency = catalogue.Update_freq.ToString(); + + hdrDataset.data.versions.First().metadata.metadata.accessibility.access.jurisdiction = catalogue.Juristiction.Split(",").ToList(); + hdrDataset.data.versions.First().metadata.metadata.accessibility.access.dataController = catalogue.DataController; + hdrDataset.data.versions.First().metadata.metadata.accessibility.access.dataProcessor = catalogue.DataProcessor; + + hdrDataset.data.versions.First().metadata.metadata.identifier = "05ec5a13-3955-45a3-b449-8aba78622113";// hdrDataset.data.versions.First().metadata.identifier; + + Update(hdrDataset.data.id.ToString(), hdrDataset); + } + + private class HDRUpdateObject + { + public int team_id { get; set; } + public int user_id { get; set; } + public string create_origin { get; set; } + public PatchSubMetadata metadata { get; set; } + } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/Helpers/CustomDateTimeConverter.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/Helpers/CustomDateTimeConverter.cs new file mode 100644 index 0000000000..5b2b294635 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/Helpers/CustomDateTimeConverter.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.Helpers +{ + /// + /// + /// + public class CustomDateTimeConverter : JsonConverter + { + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return DateTime.Parse(reader.GetString()); + } + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString("yyyy-MM-ddTHH:mm:ss.000000Z", CultureInfo.InvariantCulture)); + } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/Helpers/CustomDateTimeConverterThreeMilliseconds.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/Helpers/CustomDateTimeConverterThreeMilliseconds.cs new file mode 100644 index 0000000000..519625f5cb --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/Helpers/CustomDateTimeConverterThreeMilliseconds.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.Helpers +{ + /// + /// + /// + public class CustomDateTimeConverterThreeMilliseconds : JsonConverter + { + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return DateTime.Parse(reader.GetString()); + } + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString("yyyy-MM-ddTHH:mm:ss.000Z", CultureInfo.InvariantCulture)); + } + } +} diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index c75dc69052..93b81068a4 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -30,7 +30,14 @@ private IAtomicCommand[] GetWhitespaceRightClickMenu() new ExecuteCommandAddNewRegexRedactionConfigurationUI(_activator) { OverrideCommandName="Add New Regex Redaction Configuration" - } + }, + new ExecuteCommandCreateNewHDRConfigurationUI(_activator){ + OverrideCommandName="Create New HDR Configuration", SuggestedCategory="HDR Integration" + }, + new ExecuteCommandImportExistingHDRDatasetUI(_activator) + { + OverrideCommandName="Import Existing HDR Dataset", SuggestedCategory="HDR Integration" + }, }; } diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewHDRConfigurationUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewHDRConfigurationUI.cs new file mode 100644 index 0000000000..8b25c312a4 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewHDRConfigurationUI.cs @@ -0,0 +1,29 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.CommandExecution; +using Rdmp.UI.ItemActivation; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Rdmp.UI.SimpleDialogs.Datasets.HDR; + +namespace Rdmp.UI.CommandExecution.AtomicCommands +{ + class ExecuteCommandCreateNewHDRConfigurationUI : BasicCommandExecution, IAtomicCommand + { + private readonly IActivateItems _activator; + + public ExecuteCommandCreateNewHDRConfigurationUI(IActivateItems activator) : base(activator) + { + _activator = activator; + } + + public override void Execute() + { + base.Execute(); + var ui = new CreateNewHDRConfigurationUI(_activator); + ui.ShowDialog(); + } + } +} diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingHDRDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingHDRDatasetUI.cs new file mode 100644 index 0000000000..6c2c888659 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingHDRDatasetUI.cs @@ -0,0 +1,28 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.SimpleDialogs.Datasets.HDR; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.UI.CommandExecution.AtomicCommands +{ + public class ExecuteCommandImportExistingHDRDatasetUI : ExecuteCommandCreateDataset + { + private readonly IActivateItems _activator; + + public ExecuteCommandImportExistingHDRDatasetUI(IActivateItems activator) : base( + activator, "New Dataset") + { + _activator = activator; + } + + public override void Execute() + { + var ui = new ImportExistingHDRDatasetUI(_activator, this); + ui.ShowDialog(); + } + } +} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.Designer.cs new file mode 100644 index 0000000000..933e16f1ac --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.Designer.cs @@ -0,0 +1,250 @@ +using Org.BouncyCastle.Asn1.Crmf; +using System.Runtime.Intrinsics.Arm; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Datasets.HDR +{ + partial class CreateNewHDRConfigurationUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + label1 = new System.Windows.Forms.Label(); + label2 = new System.Windows.Forms.Label(); + label3 = new System.Windows.Forms.Label(); + label4 = new System.Windows.Forms.Label(); + tbName = new System.Windows.Forms.TextBox(); + tbUrl = new System.Windows.Forms.TextBox(); + tbOrganisationId = new System.Windows.Forms.TextBox(); + cbCredentials = new System.Windows.Forms.ComboBox(); + btnSave = new System.Windows.Forms.Button(); + cbImportCatalogues = new System.Windows.Forms.CheckBox(); + cbIncludeInternal = new System.Windows.Forms.CheckBox(); + cbImportProjectSpecific = new System.Windows.Forms.CheckBox(); + cbImportDeprecated = new System.Windows.Forms.CheckBox(); + aiImportAll = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + aiInternal = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + aiProjectSpecific = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + aiDeprecated = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + SuspendLayout(); + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(57, 76); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(104, 15); + label1.TabIndex = 0; + label1.Text = "Object Schema ID:"; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(119, 37); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(42, 15); + label2.TabIndex = 1; + label2.Text = "Name:"; + // + // label3 + // + label3.AutoSize = true; + label3.Location = new System.Drawing.Point(26, 116); + label3.Name = "label3"; + label3.Size = new System.Drawing.Size(135, 15); + label3.TabIndex = 2; + label3.Text = "Data Access Credentials:"; + // + // label4 + // + label4.AutoSize = true; + label4.Location = new System.Drawing.Point(69, 152); + label4.Name = "label4"; + label4.Size = new System.Drawing.Size(92, 15); + label4.TabIndex = 3; + label4.Text = "Organisation ID:"; + // + // tbName + // + tbName.Location = new System.Drawing.Point(169, 34); + tbName.Name = "tbName"; + tbName.Size = new System.Drawing.Size(370, 23); + tbName.TabIndex = 4; + tbName.TextChanged += ValidateForm; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(169, 73); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(370, 23); + tbUrl.TabIndex = 5; + tbUrl.TextChanged += ValidateForm; + // + // tbOrganisationId + // + tbOrganisationId.Location = new System.Drawing.Point(169, 149); + tbOrganisationId.Name = "tbOrganisationId"; + tbOrganisationId.Size = new System.Drawing.Size(370, 23); + tbOrganisationId.TabIndex = 6; + tbOrganisationId.TextChanged += ValidateForm; + // + // cbCredentials + // + cbCredentials.FormattingEnabled = true; + cbCredentials.Location = new System.Drawing.Point(169, 108); + cbCredentials.Name = "cbCredentials"; + cbCredentials.Size = new System.Drawing.Size(208, 23); + cbCredentials.TabIndex = 7; + cbCredentials.SelectedIndexChanged += ValidateForm; + // + // btnSave + // + btnSave.Enabled = false; + btnSave.Location = new System.Drawing.Point(464, 318); + btnSave.Name = "btnSave"; + btnSave.Size = new System.Drawing.Size(75, 23); + btnSave.TabIndex = 8; + btnSave.Text = "Save"; + btnSave.UseVisualStyleBackColor = true; + btnSave.Click += Save; + // + // cbImportCatalogues + // + cbImportCatalogues.AutoSize = true; + cbImportCatalogues.Location = new System.Drawing.Point(337, 188); + cbImportCatalogues.Name = "cbImportCatalogues"; + cbImportCatalogues.Size = new System.Drawing.Size(202, 19); + cbImportCatalogues.TabIndex = 9; + cbImportCatalogues.Text = "Import All Extractable Catalogues"; + cbImportCatalogues.UseVisualStyleBackColor = true; + // + // cbIncludeInternal + // + cbIncludeInternal.AutoSize = true; + cbIncludeInternal.Location = new System.Drawing.Point(337, 213); + cbIncludeInternal.Name = "cbIncludeInternal"; + cbIncludeInternal.Size = new System.Drawing.Size(184, 19); + cbIncludeInternal.TabIndex = 10; + cbIncludeInternal.Text = "Import All Internal Catalogues"; + cbIncludeInternal.UseVisualStyleBackColor = true; + // + // cbImportProjectSpecific + // + cbImportProjectSpecific.AutoSize = true; + cbImportProjectSpecific.Location = new System.Drawing.Point(337, 238); + cbImportProjectSpecific.Name = "cbImportProjectSpecific"; + cbImportProjectSpecific.Size = new System.Drawing.Size(225, 19); + cbImportProjectSpecific.TabIndex = 11; + cbImportProjectSpecific.Text = "Import All Project Sepcific Catalogues"; + cbImportProjectSpecific.UseVisualStyleBackColor = true; + // + // cbImportDeprecated + // + cbImportDeprecated.AutoSize = true; + cbImportDeprecated.Location = new System.Drawing.Point(337, 263); + cbImportDeprecated.Name = "cbImportDeprecated"; + cbImportDeprecated.Size = new System.Drawing.Size(204, 19); + cbImportDeprecated.TabIndex = 12; + cbImportDeprecated.Text = "Import All Deprecated Catalogues"; + cbImportDeprecated.UseVisualStyleBackColor = true; + // + // aiImportAll + // + aiImportAll.Location = new System.Drawing.Point(542, 188); + aiImportAll.Name = "aiImportAll"; + aiImportAll.Size = new System.Drawing.Size(20, 20); + aiImportAll.TabIndex = 13; + // + // aiInternal + // + aiInternal.Location = new System.Drawing.Point(521, 213); + aiInternal.Name = "aiInternal"; + aiInternal.Size = new System.Drawing.Size(20, 20); + aiInternal.TabIndex = 14; + // + // aiProjectSpecific + // + aiProjectSpecific.Location = new System.Drawing.Point(559, 237); + aiProjectSpecific.Name = "aiProjectSpecific"; + aiProjectSpecific.Size = new System.Drawing.Size(20, 20); + aiProjectSpecific.TabIndex = 15; + // + // aiDeprecated + // + aiDeprecated.Location = new System.Drawing.Point(542, 263); + aiDeprecated.Name = "aiDeprecated"; + aiDeprecated.Size = new System.Drawing.Size(20, 20); + aiDeprecated.TabIndex = 16; + // + // CreateNewJiraConfigurationUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(800, 450); + Controls.Add(aiDeprecated); + Controls.Add(aiProjectSpecific); + Controls.Add(aiInternal); + Controls.Add(aiImportAll); + Controls.Add(cbImportDeprecated); + Controls.Add(cbImportProjectSpecific); + Controls.Add(cbIncludeInternal); + Controls.Add(cbImportCatalogues); + Controls.Add(btnSave); + Controls.Add(cbCredentials); + Controls.Add(tbOrganisationId); + Controls.Add(tbUrl); + Controls.Add(tbName); + Controls.Add(label4); + Controls.Add(label3); + Controls.Add(label2); + Controls.Add(label1); + Name = "CreateNewHDRConfigurationUI"; + Text = "Create HDR Configuration"; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label label1; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.Label label3; + private System.Windows.Forms.Label label4; + private System.Windows.Forms.TextBox tbName; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.TextBox tbOrganisationId; + private System.Windows.Forms.ComboBox cbCredentials; + private System.Windows.Forms.Button btnSave; + private System.Windows.Forms.CheckBox cbImportCatalogues; + private System.Windows.Forms.CheckBox cbIncludeInternal; + private System.Windows.Forms.CheckBox cbImportProjectSpecific; + private System.Windows.Forms.CheckBox cbImportDeprecated; + private SimpleControls.AdditionalInfomationUI aiImportAll; + private SimpleControls.AdditionalInfomationUI aiInternal; + private SimpleControls.AdditionalInfomationUI aiProjectSpecific; + private SimpleControls.AdditionalInfomationUI aiDeprecated; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.cs new file mode 100644 index 0000000000..ec826a11ee --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.cs @@ -0,0 +1,79 @@ +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; +using Rdmp.Core.Curation.Data.Datasets.HDR; + +namespace Rdmp.UI.SimpleDialogs.Datasets.HDR +{ + public partial class CreateNewHDRConfigurationUI : RDMPForm + { + + private readonly IActivateItems _activator; + public CreateNewHDRConfigurationUI(IActivateItems activator) + { + InitializeComponent(); + _activator = activator; + var dataAccessCredentials = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects(); + cbCredentials.Items.Clear(); + cbCredentials.Items.AddRange(dataAccessCredentials); + } + + private void DisableSave() + { + btnSave.Enabled = false; + } + + private void ValidateForm(object sender, EventArgs e) + { + + if (cbCredentials.SelectedItem is null) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbName.Text)) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbUrl.Text)) + { + DisableSave(); + return; + } + if (string.IsNullOrWhiteSpace(tbOrganisationId.Text)) + { + DisableSave(); + return; + } + btnSave.Enabled = true; + } + + private void Save(object sender, EventArgs e) + { + var config = new DatasetProviderConfiguration(_activator.RepositoryLocator.CatalogueRepository, tbName.Text, typeof(HDRDatasetProvider).ToString(), tbUrl.Text, ((DataAccessCredentials)cbCredentials.SelectedItem).ID, tbOrganisationId.Text); + config.SaveToDatabase(); + _activator.Publish(config); + + if (cbImportCatalogues.Checked || cbImportProjectSpecific.Checked || cbIncludeInternal.Checked || cbImportDeprecated.Checked) + { + var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(_activator, new HDRDatasetProvider(_activator, config), cbImportCatalogues.Checked, cbIncludeInternal.Checked, cbImportProjectSpecific.Checked, cbImportDeprecated.Checked); + cmd.Execute(); + } + + Close(); + _activator.Show($"Dataset Provider '{tbName.Text}' has successfully been created"); + } + } +} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.Designer.cs new file mode 100644 index 0000000000..728584945a --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.Designer.cs @@ -0,0 +1,134 @@ +using Org.BouncyCastle.Asn1.Crmf; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Datasets.HDR +{ + partial class ImportExistingHDRDatasetUI + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + btnCreate = new System.Windows.Forms.Button(); + btnCancel = new System.Windows.Forms.Button(); + tbUrl = new System.Windows.Forms.TextBox(); + lblError = new System.Windows.Forms.Label(); + label2 = new System.Windows.Forms.Label(); + cbProviders = new System.Windows.Forms.ComboBox(); + label1 = new System.Windows.Forms.Label(); + SuspendLayout(); + // + // btnCreate + // + btnCreate.Location = new System.Drawing.Point(550, 108); + btnCreate.Name = "btnCreate"; + btnCreate.Size = new System.Drawing.Size(75, 23); + btnCreate.TabIndex = 14; + btnCreate.Text = "Create"; + btnCreate.UseVisualStyleBackColor = true; + btnCreate.Click += btnCreate_Click; + // + // btnCancel + // + btnCancel.Location = new System.Drawing.Point(459, 108); + btnCancel.Name = "btnCancel"; + btnCancel.Size = new System.Drawing.Size(75, 23); + btnCancel.TabIndex = 15; + btnCancel.Text = "Cancel"; + btnCancel.UseVisualStyleBackColor = true; + btnCancel.Click += btnCancel_Click; + // + // tbUrl + // + tbUrl.Location = new System.Drawing.Point(74, 56); + tbUrl.Name = "tbUrl"; + tbUrl.Size = new System.Drawing.Size(551, 23); + tbUrl.TabIndex = 17; + // + // lblError + // + lblError.AutoSize = true; + lblError.ForeColor = System.Drawing.Color.Red; + lblError.Location = new System.Drawing.Point(23, 95); + lblError.Name = "lblError"; + lblError.Size = new System.Drawing.Size(0, 15); + lblError.TabIndex = 18; + // + // label2 + // + label2.AutoSize = true; + label2.Location = new System.Drawing.Point(14, 20); + label2.Name = "label2"; + label2.Size = new System.Drawing.Size(54, 15); + label2.TabIndex = 19; + label2.Text = "Provider:"; + // + // cbProviders + // + cbProviders.FormattingEnabled = true; + cbProviders.Location = new System.Drawing.Point(74, 17); + cbProviders.Name = "cbProviders"; + cbProviders.Size = new System.Drawing.Size(345, 23); + cbProviders.TabIndex = 20; + cbProviders.SelectedIndexChanged += cbProviders_SelectedIndexChanged; + // + // label1 + // + label1.AutoSize = true; + label1.Location = new System.Drawing.Point(37, 59); + label1.Name = "label1"; + label1.Size = new System.Drawing.Size(37, 15); + label1.TabIndex = 16; + label1.Text = "UUID:"; + // + // ImportExistingPureDatasetUI + // + AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); + AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + ClientSize = new System.Drawing.Size(667, 158); + Controls.Add(cbProviders); + Controls.Add(label2); + Controls.Add(lblError); + Controls.Add(tbUrl); + Controls.Add(label1); + Controls.Add(btnCancel); + Controls.Add(btnCreate); + Name = "ImportExistingPureDatasetUI"; + Text = "Import Existing HDR Dataset"; + Load += ImportExistingPureDatasetUI_Load; + ResumeLayout(false); + PerformLayout(); + } + + #endregion + private System.Windows.Forms.Button btnCreate; + private System.Windows.Forms.Button btnCancel; + private System.Windows.Forms.TextBox tbUrl; + private System.Windows.Forms.Label lblError; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.ComboBox cbProviders; + private System.Windows.Forms.Label label1; + } +} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.cs new file mode 100644 index 0000000000..bc6a5edeb2 --- /dev/null +++ b/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.cs @@ -0,0 +1,76 @@ +using Microsoft.Data.SqlClient; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data.Datasets.HDR; +using Rdmp.UI.CommandExecution.AtomicCommands; +using Rdmp.UI.ItemActivation; +using Rdmp.UI.TestsAndSetup.ServicePropogation; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace Rdmp.UI.SimpleDialogs.Datasets.HDR +{ + public partial class ImportExistingHDRDatasetUI : RDMPForm + { + private readonly IActivateItems _activator; + + private readonly string[] _visibilities = { "FREE", "CAMPUS", "BACKEND", "CONFIDENTIAL" }; + + private DatasetProviderConfiguration _providerConfiguration; + private HDRDatasetProvider _datasetProvider; + + public ImportExistingHDRDatasetUI(IActivateItems activator, ExecuteCommandImportExistingHDRDatasetUI command) : base(activator) + { + _activator = activator; + InitializeComponent(); + var configs = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Type", typeof(HDRDatasetProvider).ToString()); + cbProviders.Items.AddRange(configs); + } + + private void ImportExistingPureDatasetUI_Load(object sender, EventArgs e) + { + + } + + private void btnCancel_Click(object sender, EventArgs e) + { + Dispose(); + } + + private void btnCreate_Click(object sender, EventArgs e) + { + if (_activator.YesNo("Please confirm you wish to import this Dataset", "Import Dataset")) + { + try + { + var provider = new HDRDatasetProvider(_activator, (DatasetProviderConfiguration)cbProviders.SelectedItem); + provider.AddExistingDataset(null, tbUrl.Text); + Close(); + Dispose(); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + lblError.Text = "Unable to locate dataset. Ensure UUID is correct."; + lblError.Visible = true; + return; + } + } + } + + private void cbProviders_SelectedIndexChanged(object sender, EventArgs e) + { + if (cbProviders.SelectedItem is DatasetProviderConfiguration config) + { + _providerConfiguration = config; + _datasetProvider = new HDRDatasetProvider(_activator, _providerConfiguration); + } + } + } +} diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index fe1613e925..b14f9e7cce 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -29,6 +29,7 @@ using Rdmp.Core.Curation.Data.Dashboarding; using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data.Datasets.HDR; using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Pipelines; @@ -622,6 +623,10 @@ public static T WhenIHaveA(MemoryDataExportRepository repository) where T : D { return (T)(object)new DatasetProviderConfiguration(repository.CatalogueRepository, "","","",WhenIHaveA(repository).ID,""); } + if (typeof(T) == typeof(HDRDataset)) + { + return (T)(object)new HDRDataset(repository.CatalogueRepository, "HDR Dataset"); + } throw new TestCaseNotWrittenYetException(typeof(T)); } From 58e85c3496622fbc0dc3dbb13be52989bb9a58d1 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 29 Apr 2025 12:56:31 +0100 Subject: [PATCH 02/20] working create --- .../Data/Datasets/HDR/HDRDatasetPatch.cs | 4 +- .../Data/Datasets/HDR/HDRDatasetPost.cs | 53 +++++++++++++ .../Data/Datasets/HDR/HDRDatasetProvider.cs | 79 +++++++++---------- .../Data/Datasets/PluginDatasetProvider.cs | 4 +- 4 files changed, 92 insertions(+), 48 deletions(-) create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs index 3fc3cb45e1..9cabbbcb06 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs @@ -18,7 +18,7 @@ public class PatchMetadata public PatchMetadata() { } - public PatchMetadata(Metadata existingMetadata) + public PatchMetadata(HDRDatasetItems.Metadata existingMetadata) { metadata = existingMetadata.metadata != null ? new PatchSubMetadata(existingMetadata.metadata) : null; @@ -50,7 +50,7 @@ public class PatchSubMetadata public string version { get; set; } - public PatchSubMetadata(Metadata existingMetadata) + public PatchSubMetadata(HDRDatasetItems.Metadata existingMetadata) { accessibility = existingMetadata.accessibility; observations = existingMetadata.observations; diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs new file mode 100644 index 0000000000..f7a75cf83f --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs @@ -0,0 +1,53 @@ +using Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems; +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR +{ + public class Metadata + { + public Metadata() { } + public string identifier = ""; + public string version = "1.0.0"; + public List revisions = new List(); + public string modified; + public string issued; + public Summary summary = new Summary(); + public Accessibility accessibility = new Accessibility(); + public List observations = new List(); + public Provenance provenance = new Provenance(); + } + public class HDRDatasetPost + { + + public Metadata metadata = new Metadata(); + public HDRDatasetPost(Catalogue catalogue) { + metadata.identifier = ""; + metadata.version = "1.0.0"; + metadata.revisions = new List(); + metadata.modified = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); + metadata.issued = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); + metadata.summary.title=catalogue.Name; + metadata.summary.@abstract = "";// catalogue.ShortDescription; + metadata.summary.dataCustodian = new DataCustodian(); + metadata.summary.dataCustodian.identifier= "unknown"; + metadata.summary.dataCustodian.name = "name"; + metadata.summary.dataCustodian.contactPoint = "test@example.com"; + metadata.summary.populationSize = 0; + metadata.summary.contactPoint = ""; + metadata.provenance.temporal = new Temporal(); + metadata.provenance.origin = new Origin(); + metadata.provenance.temporal.timeLag = "Variable"; + metadata.provenance.temporal.startDate = DateTime.UtcNow; + metadata.provenance.temporal.publishingFrequency = "Irregular"; + metadata.provenance.origin.datasetSubType = new List(); + metadata.provenance.origin.datasetType = new List(); + metadata.accessibility.access = new Access(); + metadata.accessibility.access.accessRights = ""; + } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs index 799fed64d5..dda39871c1 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -12,6 +12,7 @@ using Newtonsoft.Json; using Rdmp.Core.Curation.Data.Datasets.HDR.Helpers; using Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems; +using MongoDB.Bson; namespace Rdmp.Core.Curation.Data.Datasets.HDR { @@ -22,7 +23,7 @@ public class HDRDatasetProvider : PluginDatasetProvider { private HttpClient _client; private DatasetProviderConfiguration _configuration; - public HDRDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration) : base(activator, configuration) + public HDRDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration, HttpClient client = null) : base(activator, configuration) { _client = new HttpClient(); var credentials = Repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); @@ -33,28 +34,19 @@ public HDRDatasetProvider(IBasicActivateItems activator, DatasetProviderConfigur } public override Dataset AddExistingDatasetWithReturn(string name, string url) { - var response = Task.Run(async () => await _client.GetAsync(url)).Result; - if (response.StatusCode == HttpStatusCode.OK) - { - var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - HDRDataset hdrDataset = JsonConvert.DeserializeObject(detailsString); - var datasetName = string.IsNullOrWhiteSpace(name) ? hdrDataset?.data.versions?.First().metadata.metadata.summary.title : name; - var dataset = new Dataset(Repository, datasetName) - { - Url = url, - Type = ToString(), - Provider_ID = Configuration.ID, - //DigitalObjectIdentifier = hdrDataset.data.versions.First().metadata.metadata.summary.doiName.ToString(), - Folder = $"\\{Configuration.Name}", - }; - dataset.SaveToDatabase(); - Activator.Publish(dataset); - return dataset; - } - else + HDRDataset hdrDataset = (HDRDataset)FetchDatasetByID(int.Parse(url)); + var datasetName = string.IsNullOrWhiteSpace(name) ? hdrDataset?.data.versions?.First().metadata.metadata.summary.title : name; + var dataset = new Dataset(Repository, datasetName) { - throw new Exception("Cannot access dataset at provided url"); - } + Url = url, + Type = ToString(), + Provider_ID = Configuration.ID, + //DigitalObjectIdentifier = hdrDataset.data.versions.First().metadata.metadata.summary.doiName.ToString(), + Folder = $"\\{Configuration.Name}", + }; + dataset.SaveToDatabase(); + Activator.Publish(dataset); + return dataset; } public override void AddExistingDataset(string name, string url) @@ -62,38 +54,39 @@ public override void AddExistingDataset(string name, string url) AddExistingDatasetWithReturn(name, url); } + private class CreateDatasetResponse + { + public int data{ get; set; } + } + public override Dataset Create(Catalogue catalogue) { - var url = Configuration.Url + "/v1/datasets"; + var url = Configuration.Url + "/v1/integrations/datasets?input_schema=HDRUK&input_version=3.0.0"; var serializeOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, WriteIndented = true, - IncludeFields = true + IncludeFields = true, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; using var stream = new MemoryStream(); - var ds = new HDRDatasetPatch(); - ds.metadata = new PatchMetadata() - { - schemaModel = "HDRUK", - schemaVersion = "3.0.0", - metadata = new PatchSubMetadata(new Metadata() - { - summary = new Summary() - { - title = catalogue.Name - } - }) - }; - System.Text.Json.JsonSerializer.Serialize(stream, ds, serializeOptions); - var jsonString = Encoding.UTF8.GetString(stream.ToArray()); + var ds = new HDRDatasetPost(catalogue); + + var jsonString = System.Text.Json.JsonSerializer.Serialize(ds, serializeOptions); var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PostAsync(url, httpContent)).Result; - if (response.StatusCode == HttpStatusCode.OK) + if (response.StatusCode == HttpStatusCode.Created) { - + var content = Task.Run(async ()=> await response.Content.ReadAsStringAsync()).Result; + var responseJson = JsonConvert.DeserializeObject(content); + var dataset = FetchDatasetByID(responseJson.data) as HDRDataset; + //UpdateUsingCatalogue(dataset, catalogue);//todo wll have to test this + return dataset; + } + else + { + throw new Exception("q"); } - throw new Exception("q"); } @@ -249,7 +242,7 @@ public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) hdrDataset.data.versions.First().metadata.metadata.summary.title = catalogue.Name; hdrDataset.data.versions.First().metadata.metadata.summary.@abstract = catalogue.ShortDescription; hdrDataset.data.versions.First().metadata.metadata.summary.contactPoint = catalogue.Administrative_contact_email; - hdrDataset.data.versions.First().metadata.metadata.summary.keywords = catalogue.Search_keywords.Split(',').Cast().ToList(); + hdrDataset.data.versions.First().metadata.metadata.summary.keywords = (catalogue.Search_keywords??"").Split(',').Cast().ToList(); hdrDataset.data.versions.First().metadata.metadata.summary.doiName = catalogue.Doi; hdrDataset.data.versions.First().metadata.metadata.documentation.description = catalogue.Description; diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index 2a03e02a8d..eff2c3520c 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -30,13 +30,11 @@ protected PluginDatasetProvider(IBasicActivateItems activator, DatasetProviderCo public abstract Curation.Data.Datasets.Dataset AddExistingDatasetWithReturn(string name, string url); - public abstract Curation.Data.Datasets.IDataset Create(Catalogue catalogue); - public abstract void Update(string uuid, PluginDataset datasetUpdates); public abstract void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue); - Dataset IDatasetProvider.Create(Catalogue catalogue) + public virtual Dataset Create(Catalogue catalogue) { throw new NotImplementedException(); } From 3a1e161926bdec831ce9b11a53dd781e37a61fec Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 29 Apr 2025 15:47:13 +0100 Subject: [PATCH 03/20] clean up --- .../Data/Datasets/HDR/HDRDatasetProvider.cs | 6 +- .../Collections/ConfigurationsCollectionUI.cs | 15 +- ...ecuteCommandCreateNewHDRConfigurationUI.cs | 29 -- ...xecuteCommandImportExistingHDRDatasetUI.cs | 28 -- .../CreateNewHDRConfigurationUI.Designer.cs | 250 ------------------ .../HDR/CreateNewHDRConfigurationUI.cs | 79 ------ .../ImportExistingHDRDatasetUI.Designer.cs | 134 ---------- .../HDR/ImportExistingHDRDatasetUI.cs | 76 ------ 8 files changed, 7 insertions(+), 610 deletions(-) delete mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewHDRConfigurationUI.cs delete mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingHDRDatasetUI.cs delete mode 100644 Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.Designer.cs delete mode 100644 Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.cs delete mode 100644 Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.Designer.cs delete mode 100644 Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.cs diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs index dda39871c1..13f9582288 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -80,12 +80,13 @@ public override Dataset Create(Catalogue catalogue) var content = Task.Run(async ()=> await response.Content.ReadAsStringAsync()).Result; var responseJson = JsonConvert.DeserializeObject(content); var dataset = FetchDatasetByID(responseJson.data) as HDRDataset; - //UpdateUsingCatalogue(dataset, catalogue);//todo wll have to test this + //(dataset, catalogue);//todo wll have to test this return dataset; } else { - throw new Exception("q"); + var content = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + throw new Exception(content); } } @@ -145,7 +146,6 @@ public override void Update(string uuid, PluginDataset datasetUpdates) }; System.Text.Json.JsonSerializer.Serialize(stream, updateObj.metadata, serializeOptions); var jsonString = "{\"metadata\":{\"metadata\":" + Encoding.UTF8.GetString(stream.ToArray()) + "}}"; - //var uri = $"{Configuration.Url}/v1/integrations/datasets/{uuid}?schema_model=HDRUK&schema_version=3.0.0"; var uri = $"{Configuration.Url}/v1/integrations/datasets/{uuid}?input_schema=HDRUK&input_version=3.0.0"; var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); diff --git a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs index 5d2ac9c789..256bf820df 100644 --- a/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs +++ b/Rdmp.UI/Collections/ConfigurationsCollectionUI.cs @@ -35,24 +35,17 @@ private IAtomicCommand[] GetWhitespaceRightClickMenu() { OverrideCommandName="Add New Regex Redaction Configuration" }, - new ExecuteCommandCreateNewHDRConfigurationUI(_activator){ - OverrideCommandName="Create New HDR Configuration", SuggestedCategory="HDR Integration" - }, - new ExecuteCommandImportExistingHDRDatasetUI(_activator) - { - OverrideCommandName="Import Existing HDR Dataset", SuggestedCategory="HDR Integration" - }, }; - foreach(var provider in datasetProviders) + foreach (var provider in datasetProviders) { - options = options.Append(new ExecuteCommandAddNewDatasetProviderUI(_activator,provider) + options = options.Append(new ExecuteCommandAddNewDatasetProviderUI(_activator, provider) { OverrideCommandName = $"Add New {System.Text.RegularExpressions.Regex.Replace(provider.Name, "([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim()}", SuggestedCategory = "Dataset Provider Configurations" }).ToArray(); options = options.Append(new ExecuteCommandAddNewDatasetUI(_activator, provider) { - OverrideCommandName = $"Add Existing {System.Text.RegularExpressions.Regex.Replace(provider.Name, "([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim().Replace("Provider","")}", + OverrideCommandName = $"Add Existing {System.Text.RegularExpressions.Regex.Replace(provider.Name, "([A-Z])", " $1", System.Text.RegularExpressions.RegexOptions.Compiled).Trim().Replace("Provider", "")}", SuggestedCategory = "Datasets" }).ToArray(); } @@ -72,7 +65,7 @@ public override void SetItemActivator(IActivateItems activator) tlvConfigurations.AddObject(Activator.CoreChildProvider.AllDatasetProviderConfigurationsNode); tlvConfigurations.AddObject(Activator.CoreChildProvider.AllRegexRedactionConfigurationsNode); tlvConfigurations.Refresh(); - } + } public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e) { diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewHDRConfigurationUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewHDRConfigurationUI.cs deleted file mode 100644 index 8b25c312a4..0000000000 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandCreateNewHDRConfigurationUI.cs +++ /dev/null @@ -1,29 +0,0 @@ -using Rdmp.Core.CommandExecution.AtomicCommands; -using Rdmp.Core.CommandExecution; -using Rdmp.UI.ItemActivation; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Rdmp.UI.SimpleDialogs.Datasets.HDR; - -namespace Rdmp.UI.CommandExecution.AtomicCommands -{ - class ExecuteCommandCreateNewHDRConfigurationUI : BasicCommandExecution, IAtomicCommand - { - private readonly IActivateItems _activator; - - public ExecuteCommandCreateNewHDRConfigurationUI(IActivateItems activator) : base(activator) - { - _activator = activator; - } - - public override void Execute() - { - base.Execute(); - var ui = new CreateNewHDRConfigurationUI(_activator); - ui.ShowDialog(); - } - } -} diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingHDRDatasetUI.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingHDRDatasetUI.cs deleted file mode 100644 index 6c2c888659..0000000000 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandImportExistingHDRDatasetUI.cs +++ /dev/null @@ -1,28 +0,0 @@ -using Rdmp.Core.CommandExecution.AtomicCommands; -using Rdmp.UI.ItemActivation; -using Rdmp.UI.SimpleDialogs.Datasets.HDR; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Rdmp.UI.CommandExecution.AtomicCommands -{ - public class ExecuteCommandImportExistingHDRDatasetUI : ExecuteCommandCreateDataset - { - private readonly IActivateItems _activator; - - public ExecuteCommandImportExistingHDRDatasetUI(IActivateItems activator) : base( - activator, "New Dataset") - { - _activator = activator; - } - - public override void Execute() - { - var ui = new ImportExistingHDRDatasetUI(_activator, this); - ui.ShowDialog(); - } - } -} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.Designer.cs deleted file mode 100644 index 933e16f1ac..0000000000 --- a/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.Designer.cs +++ /dev/null @@ -1,250 +0,0 @@ -using Org.BouncyCastle.Asn1.Crmf; -using System.Runtime.Intrinsics.Arm; -using System.Windows.Forms; - -namespace Rdmp.UI.SimpleDialogs.Datasets.HDR -{ - partial class CreateNewHDRConfigurationUI - { - /// - /// Required designer variable. - /// - private System.ComponentModel.IContainer components = null; - - /// - /// Clean up any resources being used. - /// - /// true if managed resources should be disposed; otherwise, false. - protected override void Dispose(bool disposing) - { - if (disposing && (components != null)) - { - components.Dispose(); - } - base.Dispose(disposing); - } - - #region Windows Form Designer generated code - - /// - /// Required method for Designer support - do not modify - /// the contents of this method with the code editor. - /// - private void InitializeComponent() - { - label1 = new System.Windows.Forms.Label(); - label2 = new System.Windows.Forms.Label(); - label3 = new System.Windows.Forms.Label(); - label4 = new System.Windows.Forms.Label(); - tbName = new System.Windows.Forms.TextBox(); - tbUrl = new System.Windows.Forms.TextBox(); - tbOrganisationId = new System.Windows.Forms.TextBox(); - cbCredentials = new System.Windows.Forms.ComboBox(); - btnSave = new System.Windows.Forms.Button(); - cbImportCatalogues = new System.Windows.Forms.CheckBox(); - cbIncludeInternal = new System.Windows.Forms.CheckBox(); - cbImportProjectSpecific = new System.Windows.Forms.CheckBox(); - cbImportDeprecated = new System.Windows.Forms.CheckBox(); - aiImportAll = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); - aiInternal = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); - aiProjectSpecific = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); - aiDeprecated = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); - SuspendLayout(); - // - // label1 - // - label1.AutoSize = true; - label1.Location = new System.Drawing.Point(57, 76); - label1.Name = "label1"; - label1.Size = new System.Drawing.Size(104, 15); - label1.TabIndex = 0; - label1.Text = "Object Schema ID:"; - // - // label2 - // - label2.AutoSize = true; - label2.Location = new System.Drawing.Point(119, 37); - label2.Name = "label2"; - label2.Size = new System.Drawing.Size(42, 15); - label2.TabIndex = 1; - label2.Text = "Name:"; - // - // label3 - // - label3.AutoSize = true; - label3.Location = new System.Drawing.Point(26, 116); - label3.Name = "label3"; - label3.Size = new System.Drawing.Size(135, 15); - label3.TabIndex = 2; - label3.Text = "Data Access Credentials:"; - // - // label4 - // - label4.AutoSize = true; - label4.Location = new System.Drawing.Point(69, 152); - label4.Name = "label4"; - label4.Size = new System.Drawing.Size(92, 15); - label4.TabIndex = 3; - label4.Text = "Organisation ID:"; - // - // tbName - // - tbName.Location = new System.Drawing.Point(169, 34); - tbName.Name = "tbName"; - tbName.Size = new System.Drawing.Size(370, 23); - tbName.TabIndex = 4; - tbName.TextChanged += ValidateForm; - // - // tbUrl - // - tbUrl.Location = new System.Drawing.Point(169, 73); - tbUrl.Name = "tbUrl"; - tbUrl.Size = new System.Drawing.Size(370, 23); - tbUrl.TabIndex = 5; - tbUrl.TextChanged += ValidateForm; - // - // tbOrganisationId - // - tbOrganisationId.Location = new System.Drawing.Point(169, 149); - tbOrganisationId.Name = "tbOrganisationId"; - tbOrganisationId.Size = new System.Drawing.Size(370, 23); - tbOrganisationId.TabIndex = 6; - tbOrganisationId.TextChanged += ValidateForm; - // - // cbCredentials - // - cbCredentials.FormattingEnabled = true; - cbCredentials.Location = new System.Drawing.Point(169, 108); - cbCredentials.Name = "cbCredentials"; - cbCredentials.Size = new System.Drawing.Size(208, 23); - cbCredentials.TabIndex = 7; - cbCredentials.SelectedIndexChanged += ValidateForm; - // - // btnSave - // - btnSave.Enabled = false; - btnSave.Location = new System.Drawing.Point(464, 318); - btnSave.Name = "btnSave"; - btnSave.Size = new System.Drawing.Size(75, 23); - btnSave.TabIndex = 8; - btnSave.Text = "Save"; - btnSave.UseVisualStyleBackColor = true; - btnSave.Click += Save; - // - // cbImportCatalogues - // - cbImportCatalogues.AutoSize = true; - cbImportCatalogues.Location = new System.Drawing.Point(337, 188); - cbImportCatalogues.Name = "cbImportCatalogues"; - cbImportCatalogues.Size = new System.Drawing.Size(202, 19); - cbImportCatalogues.TabIndex = 9; - cbImportCatalogues.Text = "Import All Extractable Catalogues"; - cbImportCatalogues.UseVisualStyleBackColor = true; - // - // cbIncludeInternal - // - cbIncludeInternal.AutoSize = true; - cbIncludeInternal.Location = new System.Drawing.Point(337, 213); - cbIncludeInternal.Name = "cbIncludeInternal"; - cbIncludeInternal.Size = new System.Drawing.Size(184, 19); - cbIncludeInternal.TabIndex = 10; - cbIncludeInternal.Text = "Import All Internal Catalogues"; - cbIncludeInternal.UseVisualStyleBackColor = true; - // - // cbImportProjectSpecific - // - cbImportProjectSpecific.AutoSize = true; - cbImportProjectSpecific.Location = new System.Drawing.Point(337, 238); - cbImportProjectSpecific.Name = "cbImportProjectSpecific"; - cbImportProjectSpecific.Size = new System.Drawing.Size(225, 19); - cbImportProjectSpecific.TabIndex = 11; - cbImportProjectSpecific.Text = "Import All Project Sepcific Catalogues"; - cbImportProjectSpecific.UseVisualStyleBackColor = true; - // - // cbImportDeprecated - // - cbImportDeprecated.AutoSize = true; - cbImportDeprecated.Location = new System.Drawing.Point(337, 263); - cbImportDeprecated.Name = "cbImportDeprecated"; - cbImportDeprecated.Size = new System.Drawing.Size(204, 19); - cbImportDeprecated.TabIndex = 12; - cbImportDeprecated.Text = "Import All Deprecated Catalogues"; - cbImportDeprecated.UseVisualStyleBackColor = true; - // - // aiImportAll - // - aiImportAll.Location = new System.Drawing.Point(542, 188); - aiImportAll.Name = "aiImportAll"; - aiImportAll.Size = new System.Drawing.Size(20, 20); - aiImportAll.TabIndex = 13; - // - // aiInternal - // - aiInternal.Location = new System.Drawing.Point(521, 213); - aiInternal.Name = "aiInternal"; - aiInternal.Size = new System.Drawing.Size(20, 20); - aiInternal.TabIndex = 14; - // - // aiProjectSpecific - // - aiProjectSpecific.Location = new System.Drawing.Point(559, 237); - aiProjectSpecific.Name = "aiProjectSpecific"; - aiProjectSpecific.Size = new System.Drawing.Size(20, 20); - aiProjectSpecific.TabIndex = 15; - // - // aiDeprecated - // - aiDeprecated.Location = new System.Drawing.Point(542, 263); - aiDeprecated.Name = "aiDeprecated"; - aiDeprecated.Size = new System.Drawing.Size(20, 20); - aiDeprecated.TabIndex = 16; - // - // CreateNewJiraConfigurationUI - // - AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); - AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - ClientSize = new System.Drawing.Size(800, 450); - Controls.Add(aiDeprecated); - Controls.Add(aiProjectSpecific); - Controls.Add(aiInternal); - Controls.Add(aiImportAll); - Controls.Add(cbImportDeprecated); - Controls.Add(cbImportProjectSpecific); - Controls.Add(cbIncludeInternal); - Controls.Add(cbImportCatalogues); - Controls.Add(btnSave); - Controls.Add(cbCredentials); - Controls.Add(tbOrganisationId); - Controls.Add(tbUrl); - Controls.Add(tbName); - Controls.Add(label4); - Controls.Add(label3); - Controls.Add(label2); - Controls.Add(label1); - Name = "CreateNewHDRConfigurationUI"; - Text = "Create HDR Configuration"; - ResumeLayout(false); - PerformLayout(); - } - - #endregion - - private System.Windows.Forms.Label label1; - private System.Windows.Forms.Label label2; - private System.Windows.Forms.Label label3; - private System.Windows.Forms.Label label4; - private System.Windows.Forms.TextBox tbName; - private System.Windows.Forms.TextBox tbUrl; - private System.Windows.Forms.TextBox tbOrganisationId; - private System.Windows.Forms.ComboBox cbCredentials; - private System.Windows.Forms.Button btnSave; - private System.Windows.Forms.CheckBox cbImportCatalogues; - private System.Windows.Forms.CheckBox cbIncludeInternal; - private System.Windows.Forms.CheckBox cbImportProjectSpecific; - private System.Windows.Forms.CheckBox cbImportDeprecated; - private SimpleControls.AdditionalInfomationUI aiImportAll; - private SimpleControls.AdditionalInfomationUI aiInternal; - private SimpleControls.AdditionalInfomationUI aiProjectSpecific; - private SimpleControls.AdditionalInfomationUI aiDeprecated; - } -} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.cs deleted file mode 100644 index ec826a11ee..0000000000 --- a/Rdmp.UI/SimpleDialogs/Datasets/HDR/CreateNewHDRConfigurationUI.cs +++ /dev/null @@ -1,79 +0,0 @@ -using Rdmp.Core.CommandExecution.AtomicCommands; -using Rdmp.Core.Curation.Data.Datasets; -using Rdmp.Core.Curation.Data; -using Rdmp.UI.ItemActivation; -using Rdmp.UI.TestsAndSetup.ServicePropogation; -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Data; -using System.Drawing; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows.Forms; -using Rdmp.Core.Curation.Data.Datasets.HDR; - -namespace Rdmp.UI.SimpleDialogs.Datasets.HDR -{ - public partial class CreateNewHDRConfigurationUI : RDMPForm - { - - private readonly IActivateItems _activator; - public CreateNewHDRConfigurationUI(IActivateItems activator) - { - InitializeComponent(); - _activator = activator; - var dataAccessCredentials = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects(); - cbCredentials.Items.Clear(); - cbCredentials.Items.AddRange(dataAccessCredentials); - } - - private void DisableSave() - { - btnSave.Enabled = false; - } - - private void ValidateForm(object sender, EventArgs e) - { - - if (cbCredentials.SelectedItem is null) - { - DisableSave(); - return; - } - if (string.IsNullOrWhiteSpace(tbName.Text)) - { - DisableSave(); - return; - } - if (string.IsNullOrWhiteSpace(tbUrl.Text)) - { - DisableSave(); - return; - } - if (string.IsNullOrWhiteSpace(tbOrganisationId.Text)) - { - DisableSave(); - return; - } - btnSave.Enabled = true; - } - - private void Save(object sender, EventArgs e) - { - var config = new DatasetProviderConfiguration(_activator.RepositoryLocator.CatalogueRepository, tbName.Text, typeof(HDRDatasetProvider).ToString(), tbUrl.Text, ((DataAccessCredentials)cbCredentials.SelectedItem).ID, tbOrganisationId.Text); - config.SaveToDatabase(); - _activator.Publish(config); - - if (cbImportCatalogues.Checked || cbImportProjectSpecific.Checked || cbIncludeInternal.Checked || cbImportDeprecated.Checked) - { - var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(_activator, new HDRDatasetProvider(_activator, config), cbImportCatalogues.Checked, cbIncludeInternal.Checked, cbImportProjectSpecific.Checked, cbImportDeprecated.Checked); - cmd.Execute(); - } - - Close(); - _activator.Show($"Dataset Provider '{tbName.Text}' has successfully been created"); - } - } -} diff --git a/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.Designer.cs b/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.Designer.cs deleted file mode 100644 index 728584945a..0000000000 --- a/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.Designer.cs +++ /dev/null @@ -1,134 +0,0 @@ -using Org.BouncyCastle.Asn1.Crmf; -using System.Windows.Forms; - -namespace Rdmp.UI.SimpleDialogs.Datasets.HDR -{ - partial class ImportExistingHDRDatasetUI - { - /// - /// Required designer variable. - /// - private System.ComponentModel.IContainer components = null; - - /// - /// Clean up any resources being used. - /// - /// true if managed resources should be disposed; otherwise, false. - protected override void Dispose(bool disposing) - { - if (disposing && (components != null)) - { - components.Dispose(); - } - base.Dispose(disposing); - } - - #region Windows Form Designer generated code - - /// - /// Required method for Designer support - do not modify - /// the contents of this method with the code editor. - /// - private void InitializeComponent() - { - btnCreate = new System.Windows.Forms.Button(); - btnCancel = new System.Windows.Forms.Button(); - tbUrl = new System.Windows.Forms.TextBox(); - lblError = new System.Windows.Forms.Label(); - label2 = new System.Windows.Forms.Label(); - cbProviders = new System.Windows.Forms.ComboBox(); - label1 = new System.Windows.Forms.Label(); - SuspendLayout(); - // - // btnCreate - // - btnCreate.Location = new System.Drawing.Point(550, 108); - btnCreate.Name = "btnCreate"; - btnCreate.Size = new System.Drawing.Size(75, 23); - btnCreate.TabIndex = 14; - btnCreate.Text = "Create"; - btnCreate.UseVisualStyleBackColor = true; - btnCreate.Click += btnCreate_Click; - // - // btnCancel - // - btnCancel.Location = new System.Drawing.Point(459, 108); - btnCancel.Name = "btnCancel"; - btnCancel.Size = new System.Drawing.Size(75, 23); - btnCancel.TabIndex = 15; - btnCancel.Text = "Cancel"; - btnCancel.UseVisualStyleBackColor = true; - btnCancel.Click += btnCancel_Click; - // - // tbUrl - // - tbUrl.Location = new System.Drawing.Point(74, 56); - tbUrl.Name = "tbUrl"; - tbUrl.Size = new System.Drawing.Size(551, 23); - tbUrl.TabIndex = 17; - // - // lblError - // - lblError.AutoSize = true; - lblError.ForeColor = System.Drawing.Color.Red; - lblError.Location = new System.Drawing.Point(23, 95); - lblError.Name = "lblError"; - lblError.Size = new System.Drawing.Size(0, 15); - lblError.TabIndex = 18; - // - // label2 - // - label2.AutoSize = true; - label2.Location = new System.Drawing.Point(14, 20); - label2.Name = "label2"; - label2.Size = new System.Drawing.Size(54, 15); - label2.TabIndex = 19; - label2.Text = "Provider:"; - // - // cbProviders - // - cbProviders.FormattingEnabled = true; - cbProviders.Location = new System.Drawing.Point(74, 17); - cbProviders.Name = "cbProviders"; - cbProviders.Size = new System.Drawing.Size(345, 23); - cbProviders.TabIndex = 20; - cbProviders.SelectedIndexChanged += cbProviders_SelectedIndexChanged; - // - // label1 - // - label1.AutoSize = true; - label1.Location = new System.Drawing.Point(37, 59); - label1.Name = "label1"; - label1.Size = new System.Drawing.Size(37, 15); - label1.TabIndex = 16; - label1.Text = "UUID:"; - // - // ImportExistingPureDatasetUI - // - AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); - AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - ClientSize = new System.Drawing.Size(667, 158); - Controls.Add(cbProviders); - Controls.Add(label2); - Controls.Add(lblError); - Controls.Add(tbUrl); - Controls.Add(label1); - Controls.Add(btnCancel); - Controls.Add(btnCreate); - Name = "ImportExistingPureDatasetUI"; - Text = "Import Existing HDR Dataset"; - Load += ImportExistingPureDatasetUI_Load; - ResumeLayout(false); - PerformLayout(); - } - - #endregion - private System.Windows.Forms.Button btnCreate; - private System.Windows.Forms.Button btnCancel; - private System.Windows.Forms.TextBox tbUrl; - private System.Windows.Forms.Label lblError; - private System.Windows.Forms.Label label2; - private System.Windows.Forms.ComboBox cbProviders; - private System.Windows.Forms.Label label1; - } -} \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.cs b/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.cs deleted file mode 100644 index bc6a5edeb2..0000000000 --- a/Rdmp.UI/SimpleDialogs/Datasets/HDR/ImportExistingHDRDatasetUI.cs +++ /dev/null @@ -1,76 +0,0 @@ -using Microsoft.Data.SqlClient; -using Rdmp.Core.Curation.Data.Datasets; -using Rdmp.Core.Curation.Data.Datasets.HDR; -using Rdmp.UI.CommandExecution.AtomicCommands; -using Rdmp.UI.ItemActivation; -using Rdmp.UI.TestsAndSetup.ServicePropogation; -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Data; -using System.Drawing; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows.Forms; - -namespace Rdmp.UI.SimpleDialogs.Datasets.HDR -{ - public partial class ImportExistingHDRDatasetUI : RDMPForm - { - private readonly IActivateItems _activator; - - private readonly string[] _visibilities = { "FREE", "CAMPUS", "BACKEND", "CONFIDENTIAL" }; - - private DatasetProviderConfiguration _providerConfiguration; - private HDRDatasetProvider _datasetProvider; - - public ImportExistingHDRDatasetUI(IActivateItems activator, ExecuteCommandImportExistingHDRDatasetUI command) : base(activator) - { - _activator = activator; - InitializeComponent(); - var configs = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Type", typeof(HDRDatasetProvider).ToString()); - cbProviders.Items.AddRange(configs); - } - - private void ImportExistingPureDatasetUI_Load(object sender, EventArgs e) - { - - } - - private void btnCancel_Click(object sender, EventArgs e) - { - Dispose(); - } - - private void btnCreate_Click(object sender, EventArgs e) - { - if (_activator.YesNo("Please confirm you wish to import this Dataset", "Import Dataset")) - { - try - { - var provider = new HDRDatasetProvider(_activator, (DatasetProviderConfiguration)cbProviders.SelectedItem); - provider.AddExistingDataset(null, tbUrl.Text); - Close(); - Dispose(); - } - catch (Exception ex) - { - Console.WriteLine(ex.Message); - lblError.Text = "Unable to locate dataset. Ensure UUID is correct."; - lblError.Visible = true; - return; - } - } - } - - private void cbProviders_SelectedIndexChanged(object sender, EventArgs e) - { - if (cbProviders.SelectedItem is DatasetProviderConfiguration config) - { - _providerConfiguration = config; - _datasetProvider = new HDRDatasetProvider(_activator, _providerConfiguration); - } - } - } -} From 8cf28f9d07c2339545113c06a4876032ed28efa2 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Apr 2025 08:38:56 +0100 Subject: [PATCH 04/20] working update --- .../Data/Datasets/HDR/HDRDatasetProvider.cs | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs index 13f9582288..220026d1e7 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -56,7 +56,7 @@ public override void AddExistingDataset(string name, string url) private class CreateDatasetResponse { - public int data{ get; set; } + public int data { get; set; } } public override Dataset Create(Catalogue catalogue) @@ -71,15 +71,15 @@ public override Dataset Create(Catalogue catalogue) }; using var stream = new MemoryStream(); var ds = new HDRDatasetPost(catalogue); - + var jsonString = System.Text.Json.JsonSerializer.Serialize(ds, serializeOptions); var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PostAsync(url, httpContent)).Result; if (response.StatusCode == HttpStatusCode.Created) { - var content = Task.Run(async ()=> await response.Content.ReadAsStringAsync()).Result; + var content = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; var responseJson = JsonConvert.DeserializeObject(content); - var dataset = FetchDatasetByID(responseJson.data) as HDRDataset; + var dataset = FetchDatasetByID(responseJson.data) as HDRDataset; //(dataset, catalogue);//todo wll have to test this return dataset; } @@ -238,11 +238,11 @@ private string MapTimeLagToHDR(string tl) public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) { - var hdrDataset = (HDRDataset)dataset; + var hdrDataset = (HDRDataset)FetchDatasetByID(int.Parse(dataset.Url)); hdrDataset.data.versions.First().metadata.metadata.summary.title = catalogue.Name; - hdrDataset.data.versions.First().metadata.metadata.summary.@abstract = catalogue.ShortDescription; + hdrDataset.data.versions.First().metadata.metadata.summary.@abstract = catalogue.ShortDescription.Length <5? catalogue.ShortDescription.PadRight(5): catalogue.ShortDescription; hdrDataset.data.versions.First().metadata.metadata.summary.contactPoint = catalogue.Administrative_contact_email; - hdrDataset.data.versions.First().metadata.metadata.summary.keywords = (catalogue.Search_keywords??"").Split(',').Cast().ToList(); + hdrDataset.data.versions.First().metadata.metadata.summary.keywords = (catalogue.Search_keywords ?? "").Split(',').Cast().Where(k => k != "").Cast().ToList(); hdrDataset.data.versions.First().metadata.metadata.summary.doiName = catalogue.Doi; hdrDataset.data.versions.First().metadata.metadata.documentation.description = catalogue.Description; @@ -250,14 +250,17 @@ public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) hdrDataset.data.versions.First().metadata.metadata.coverage.spatial = catalogue.Geographical_coverage; - hdrDataset.data.versions.First().metadata.metadata.provenance.origin.datasetType = catalogue.DataType.Split(",").Select(MapDataTypeToHDRDataType).ToList(); - hdrDataset.data.versions.First().metadata.metadata.provenance.origin.datasetSubType = catalogue.DataSubType.Split(",").Select(MapDataSubTypeToHDR).ToList(); - hdrDataset.data.versions.First().metadata.metadata.provenance.temporal.endDate = catalogue.EndDate.Value; - hdrDataset.data.versions.First().metadata.metadata.provenance.temporal.startDate = catalogue.StartDate.Value; + if(catalogue.DataType != null) + hdrDataset.data.versions.First().metadata.metadata.provenance.origin.datasetType = catalogue.DataType.Split(",").Select(MapDataTypeToHDRDataType).ToList(); + if (catalogue.DataSubType != null) + hdrDataset.data.versions.First().metadata.metadata.provenance.origin.datasetSubType = catalogue.DataSubType.Split(",").Select(MapDataSubTypeToHDR).ToList(); + hdrDataset.data.versions.First().metadata.metadata.provenance.temporal.endDate = catalogue.EndDate != null ? catalogue.EndDate.Value : null; + if (catalogue.StartDate != null) + hdrDataset.data.versions.First().metadata.metadata.provenance.temporal.startDate = catalogue.StartDate.Value; hdrDataset.data.versions.First().metadata.metadata.provenance.temporal.timeLag = MapTimeLagToHDR(catalogue.UpdateLag.ToString()); hdrDataset.data.versions.First().metadata.metadata.provenance.temporal.publishingFrequency = catalogue.Update_freq.ToString(); - hdrDataset.data.versions.First().metadata.metadata.accessibility.access.jurisdiction = catalogue.Juristiction.Split(",").ToList(); + hdrDataset.data.versions.First().metadata.metadata.accessibility.access.jurisdiction = (catalogue.Juristiction ?? "").Split(",").Where(k => k != "").ToList(); hdrDataset.data.versions.First().metadata.metadata.accessibility.access.dataController = catalogue.DataController; hdrDataset.data.versions.First().metadata.metadata.accessibility.access.dataProcessor = catalogue.DataProcessor; From 72f51d5f08f141b8c50ed08edad9ada404f1bf88 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Apr 2025 13:21:10 +0100 Subject: [PATCH 05/20] split files --- .../HDR/HDRDatasetItems/PatchMetadata.cs | 26 +++++++++++++++++++ .../HDR/HDRDatasetItems/PostMetadata.cs | 23 ++++++++++++++++ .../Data/Datasets/HDR/HDRDatasetPatch.cs | 16 ------------ .../Data/Datasets/HDR/HDRDatasetPost.cs | 17 ++---------- 4 files changed, 51 insertions(+), 31 deletions(-) create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs new file mode 100644 index 0000000000..9772b884f9 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + public class PatchMetadata + { + + public string schemaModel { get; set; } + public string schemaVersion { get; set; } + public PatchSubMetadata metadata { get; set; } + + + public PatchMetadata() { } + public PatchMetadata(HDRDatasetItems.Metadata existingMetadata) + { + + metadata = existingMetadata.metadata != null ? new PatchSubMetadata(existingMetadata.metadata) : null; + schemaModel = "HDRUK"; + schemaVersion = "3.0.0"; + } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs new file mode 100644 index 0000000000..92325b2e4d --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + public class PostMetadata + { + + public PostMetadata() { } + public string identifier = ""; + public string version = "1.0.0"; + public List revisions = new List(); + public string modified; + public string issued; + public Summary summary = new Summary(); + public Accessibility accessibility = new Accessibility(); + public List observations = new List(); + public Provenance provenance = new Provenance(); + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs index 9cabbbcb06..3cae640601 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs @@ -9,24 +9,8 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR { - public class PatchMetadata - { - - public string schemaModel { get; set; } - public string schemaVersion { get; set; } - public PatchSubMetadata metadata { get; set; } - public PatchMetadata() { } - public PatchMetadata(HDRDatasetItems.Metadata existingMetadata) - { - - metadata = existingMetadata.metadata != null ? new PatchSubMetadata(existingMetadata.metadata) : null; - schemaModel = "HDRUK"; - schemaVersion = "3.0.0"; - } - } - public class PatchSubMetadata { public object observations { get; set; } diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs index f7a75cf83f..cd03e9dba8 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs @@ -8,23 +8,10 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR { - public class Metadata - { - public Metadata() { } - public string identifier = ""; - public string version = "1.0.0"; - public List revisions = new List(); - public string modified; - public string issued; - public Summary summary = new Summary(); - public Accessibility accessibility = new Accessibility(); - public List observations = new List(); - public Provenance provenance = new Provenance(); - } public class HDRDatasetPost { - public Metadata metadata = new Metadata(); + public PostMetadata metadata = new PostMetadata(); public HDRDatasetPost(Catalogue catalogue) { metadata.identifier = ""; metadata.version = "1.0.0"; @@ -32,7 +19,7 @@ public HDRDatasetPost(Catalogue catalogue) { metadata.modified = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); metadata.issued = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); metadata.summary.title=catalogue.Name; - metadata.summary.@abstract = "";// catalogue.ShortDescription; + metadata.summary.@abstract = catalogue.ShortDescription.Length>4?catalogue.ShortDescription:"";// catalogue.ShortDescription; metadata.summary.dataCustodian = new DataCustodian(); metadata.summary.dataCustodian.identifier= "unknown"; metadata.summary.dataCustodian.name = "name"; From 7aa83ca102eb0e2fda722ac7b3509a32bcf9c254 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Apr 2025 13:29:27 +0100 Subject: [PATCH 06/20] tidy up --- .../Data/Datasets/HDR/HDRDatasetPatch.cs | 18 ------------------ .../Data/Datasets/HDR/HDRDatasetPost.cs | 2 +- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs index 3cae640601..0a6dfdd466 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs @@ -134,28 +134,10 @@ public HDRDatasetPatch(HDRDataset existingDataset) named_entities = existingDataset.data.named_entities; collections = existingDataset.data.collections; team = existingDataset.data.team; - //metadata = new PatchMetadata(existingDataset.data.metadata); metadata = new PatchMetadata(existingDataset.data.versions.First().metadata); - //if(metadata.metadata.accessibility.access.jurisdiction is string) - // metadata.metadata.accessibility.access.jurisdiction = metadata.metadata.accessibility.access.jurisdiction.ToString().Split(";,;"); - - //if (metadata.metadata.accessibility.formatAndStandards.conformsTo is string) - // metadata.metadata.accessibility.formatAndStandards.conformsTo = metadata.metadata.accessibility.formatAndStandards.conformsTo.ToString().Split(";,;"); - //if (metadata.metadata.accessibility.formatAndStandards.formats is string) - // metadata.metadata.accessibility.formatAndStandards.formats = metadata.metadata.accessibility.formatAndStandards.formats.ToString().Split(";,;"); - //if (metadata.metadata.accessibility.formatAndStandards.languages is string) - // metadata.metadata.accessibility.formatAndStandards.languages = metadata.metadata.accessibility.formatAndStandards.languages.ToString().Split(";,;"); - //if (metadata.metadata.accessibility.formatAndStandards.vocabularyEncodingSchemes is string) - // metadata.metadata.accessibility.formatAndStandards.vocabularyEncodingSchemes = metadata.metadata.accessibility.formatAndStandards.vocabularyEncodingSchemes.ToString().Split(";,;"); - - //if (metadata.metadata.accessibility.usage.dataUseLimitation is string) - // metadata.metadata.accessibility.usage.dataUseLimitation = metadata.metadata.accessibility.usage.dataUseLimitation.ToString().Split(";,;"); if (metadata.metadata.accessibility.usage.dataUseRequirements is null) metadata.metadata.accessibility.usage.dataUseRequirements = []; - metadata.metadata.coverage.materialType = new List() { "Other" }; - //metadata.metadata.summary.dataCustodian.description = "Test"; - //metadata.metadata.summary.dataCustodian.logo = "https://cdn.britannica.com/92/100692-050-5B69B59B/Mallard.jpg"; } } } diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs index cd03e9dba8..a045b31b89 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs @@ -19,7 +19,7 @@ public HDRDatasetPost(Catalogue catalogue) { metadata.modified = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); metadata.issued = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); metadata.summary.title=catalogue.Name; - metadata.summary.@abstract = catalogue.ShortDescription.Length>4?catalogue.ShortDescription:"";// catalogue.ShortDescription; + metadata.summary.@abstract = catalogue.ShortDescription.Length>4?catalogue.ShortDescription:""; metadata.summary.dataCustodian = new DataCustodian(); metadata.summary.dataCustodian.identifier= "unknown"; metadata.summary.dataCustodian.name = "name"; From a3670839fbeed67eafb5999cf5b2ffc778722484 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Apr 2025 13:32:43 +0100 Subject: [PATCH 07/20] fix override --- Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs index 220026d1e7..1b590fefa9 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -118,11 +118,6 @@ public override Dataset FetchDatasetByID(int id) throw new Exception("Unable to fetch HDR dataset"); } - public override List FetchDatasets() - { - throw new NotImplementedException(); - } - public override void Update(string uuid, PluginDataset datasetUpdates) { var serializeOptions = new JsonSerializerOptions From 9b79fc685cf1325676da823fdd29c4de0c13257f Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Apr 2025 13:36:23 +0100 Subject: [PATCH 08/20] tidy up --- Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs index 1b590fefa9..19d1b6c5ac 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -80,7 +80,7 @@ public override Dataset Create(Catalogue catalogue) var content = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; var responseJson = JsonConvert.DeserializeObject(content); var dataset = FetchDatasetByID(responseJson.data) as HDRDataset; - //(dataset, catalogue);//todo wll have to test this + UpdateUsingCatalogue(dataset, catalogue); return dataset; } else From 70dbb0a3df2615253c8f0bc94f7aa4147db4f190 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Apr 2025 13:45:25 +0100 Subject: [PATCH 09/20] split files --- .../HDR/HDRDatasetItems/PatchSubMetadata.cs | 51 +++++++++++++++++++ .../Data/Datasets/HDR/HDRDatasetPatch.cs | 42 --------------- 2 files changed, 51 insertions(+), 42 deletions(-) create mode 100644 Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs new file mode 100644 index 0000000000..29b04a4612 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs @@ -0,0 +1,51 @@ +using Rdmp.Core.Curation.Data.Datasets.HDR.Helpers; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems +{ + public class PatchSubMetadata + { + public object observations { get; set; } + public Coverage coverage { get; set; } + public object structuralMetadata { get; set; } + public object enrichmentAndLinkage { get; set; } + public Accessibility accessibility { get; set; } + + public string identifier { get; set; } + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + + public DateTime issued { get; set; } + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + + public DateTime modified { get; set; } + + public Provenance provenance { get; set; } + public object documentation { get; set; } + public Summary summary { get; set; } + public List revisions { get; set; } + + public string version { get; set; } + + public PatchSubMetadata(HDRDatasetItems.Metadata existingMetadata) + { + accessibility = existingMetadata.accessibility; + observations = existingMetadata.observations; + coverage = existingMetadata.coverage; + structuralMetadata = existingMetadata.structuralMetadata; + enrichmentAndLinkage = existingMetadata.enrichmentAndLinkage; + provenance = existingMetadata.provenance; + documentation = existingMetadata.documentation; + summary = existingMetadata.summary; + identifier = existingMetadata.identifier; + issued = existingMetadata.issued; + modified = existingMetadata.modified; + revisions = existingMetadata.revisions; + version = existingMetadata.version; + } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs index 0a6dfdd466..12e12c0d3b 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs @@ -10,48 +10,6 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR { - - public class PatchSubMetadata - { - public object observations { get; set; } - public Coverage coverage { get; set; } - public object structuralMetadata { get; set; } - public object enrichmentAndLinkage { get; set; } - public Accessibility accessibility { get; set; } - - public string identifier { get; set; } - [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] - - public DateTime issued { get; set; } - [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] - - public DateTime modified { get; set; } - - public Provenance provenance { get; set; } - public object documentation { get; set; } - public Summary summary { get; set; } - public List revisions { get; set; } - - public string version { get; set; } - - public PatchSubMetadata(HDRDatasetItems.Metadata existingMetadata) - { - accessibility = existingMetadata.accessibility; - observations = existingMetadata.observations; - coverage = existingMetadata.coverage; - structuralMetadata = existingMetadata.structuralMetadata; - enrichmentAndLinkage = existingMetadata.enrichmentAndLinkage; - provenance = existingMetadata.provenance; - documentation = existingMetadata.documentation; - summary = existingMetadata.summary; - identifier = existingMetadata.identifier; - issued = existingMetadata.issued; - modified = existingMetadata.modified; - revisions = existingMetadata.revisions; - version = existingMetadata.version; - } - } - public class HDRDatasetPatch { public int id { get; set; } From 7eb479a0972f35273a82c647710c6549f8534f13 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Apr 2025 15:22:20 +0100 Subject: [PATCH 10/20] add hdr tests --- Directory.Packages.props | 1 + .../Datasets/HDR/HDRDatasetProviderTests.cs | 199 ++++++++++++++++++ Rdmp.Core.Tests/Rdmp.Core.Tests.csproj | 1 + ...ngCataloguesIntoExternalDatasetProvider.cs | 10 +- .../Curation/Data/Datasets/HDR/HDRDataset.cs | 9 + .../HDR/HDRDatasetItems/PatchSubMetadata.cs | 3 + .../HDR/HDRDatasetItems/PostMetadata.cs | 3 + .../Data/Datasets/HDR/HDRDatasetPatch.cs | 4 +- .../Data/Datasets/HDR/HDRDatasetProvider.cs | 18 +- 9 files changed, 238 insertions(+), 10 deletions(-) create mode 100644 Rdmp.Core.Tests/Datasets/HDR/HDRDatasetProviderTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 3541aa23c9..9df0d34f40 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -10,6 +10,7 @@ + diff --git a/Rdmp.Core.Tests/Datasets/HDR/HDRDatasetProviderTests.cs b/Rdmp.Core.Tests/Datasets/HDR/HDRDatasetProviderTests.cs new file mode 100644 index 0000000000..2f2cc1e1ae --- /dev/null +++ b/Rdmp.Core.Tests/Datasets/HDR/HDRDatasetProviderTests.cs @@ -0,0 +1,199 @@ +using NUnit.Framework; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Tests.Common; +using Rdmp.Core.Curation.Data.Datasets.HDR; +using JustEat.HttpClientInterception; + +namespace Rdmp.Core.Tests.Datasets.HDR +{ + class HDRDatasetProviderTests : DatabaseTests + { + private DatasetProviderConfiguration _configuration; + private HDRDatasetProvider _provider; + private Catalogue _catalogue; + private HttpClient _mockHttp; + private HttpClientInterceptorOptions options = new HttpClientInterceptorOptions { ThrowOnMissingRegistration = true }; + + [TearDown] + public void OneTimeTeardown() + { + _mockHttp.Dispose(); + } + + [OneTimeSetUp] + public void OneTimeSetup() + { + var dataAccessCredentials = new DataAccessCredentials(CatalogueRepository, "name") + { + Username = "test", + Password = "test" + }; + dataAccessCredentials.SaveToDatabase(); + + _configuration = new DatasetProviderConfiguration(CatalogueRepository, "Test Provider", typeof(HDRDatasetProvider).ToString(), "https://doesNotExist.preprod.hdruk.cloud/api", dataAccessCredentials.ID, "1234"); + + _configuration.SaveToDatabase(); + + _provider = new HDRDatasetProvider(new ThrowImmediatelyActivator(RepositoryLocator), _configuration, _mockHttp); + _catalogue = new Catalogue(CatalogueRepository, "HDR test Catalogue"); + _catalogue.SaveToDatabase(); + + } + private void SetupHttpClient(HttpRequestInterceptionBuilder builder) + { + builder.RegisterWith(options); + _mockHttp = options.CreateHttpClient(); + _provider = new HDRDatasetProvider(new ThrowImmediatelyActivator(RepositoryLocator), _configuration, _mockHttp); + } + + [Test] + public void CreateProviderTest() + { + Assert.DoesNotThrow(() => _configuration.GetProviderInstance(new ThrowImmediatelyActivator(RepositoryLocator))); + } + [Test] + public void AddExistingDatasetTest_Success() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("doesnotexist.preprod.hdruk.cloud") + .ForPath("api/v1/datasets/999") + .ForQuery("schema_model=HDRUK&schema_version=3.0.0") + .Responds() + .WithStatus(200).WithJsonContent(SucessResponseString); + SetupHttpClient(builder); + string name = "AddExistingDatasetTest_Success"; + Assert.DoesNotThrow(() => _provider.AddExistingDataset(name, url)); + var dataset = CatalogueRepository.GetAllObjects().FirstOrDefault(d => d.Name == name); + Assert.That(dataset, Is.Not.Null); + dataset.DeleteInDatabase(); + } + [Test] + public void AddExistingDatasetTest_BadRemote() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("doesnotexist.preprod.hdruk.cloud") + .ForPath("api/v1/datasets/999") + .ForQuery("schema_model=HDRUK&schema_version=3.0.0") + .Responds() + .WithStatus(400); + SetupHttpClient(builder); + string name = "HDRAddExistingDatasetTest_BadRemote"; + Assert.Throws(() => _provider.AddExistingDataset(name, url)); + var dataset = CatalogueRepository.GetAllObjects().FirstOrDefault(d => d.Name == name); + Assert.That(dataset, Is.Null); + } + + [Test] + public void AddExistingDatasetWithReturnTest_Success() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("doesnotexist.preprod.hdruk.cloud") + .ForPath("api/v1/datasets/999") + .ForQuery("schema_model=HDRUK&schema_version=3.0.0") + .Responds() + .WithStatus(200).WithJsonContent(SucessResponseString); + SetupHttpClient(builder); + string name = "AddExistingDatasetTest_Success"; + Assert.DoesNotThrow(() => _provider.AddExistingDatasetWithReturn(name, url)); + var dataset = CatalogueRepository.GetAllObjects().FirstOrDefault(d => d.Name == name); + Assert.That(dataset, Is.Not.Null); + dataset.DeleteInDatabase(); + } + [Test] + public void AddExistingDatasetWithReturnTest_BadRemote() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("doesnotexist.preprod.hdruk.cloud") + .ForPath("api/v1/datasets/999") + .ForQuery("schema_model=HDRUK&schema_version=3.0.0") + .Responds() + .WithStatus(400); + SetupHttpClient(builder); + string name = "HDRAddExistingDatasetTest_BadRemote"; + Assert.Throws(() => _provider.AddExistingDatasetWithReturn(name, url)); + var dataset = CatalogueRepository.GetAllObjects().FirstOrDefault(d => d.Name == name); + Assert.That(dataset, Is.Null); + } + + //FetchDatasetByID + [Test] + public void FetchDatasetByIDTest_Success() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("doesnotexist.preprod.hdruk.cloud") + .ForPath("api/v1/datasets/999") + .ForQuery("schema_model=HDRUK&schema_version=3.0.0") + .Responds() + .WithStatus(200).WithJsonContent(SucessResponseString); + SetupHttpClient(builder); + Assert.DoesNotThrow(() => _provider.FetchDatasetByID(int.Parse(url))); + } + [Test] + public void FetchDatasetByIDTest_BadRemote() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("doesnotexist.preprod.hdruk.cloud") + .ForPath("api/v1/datasets/999") + .ForQuery("schema_model=HDRUK&schema_version=3.0.0") + .Responds() + .WithStatus(400); + SetupHttpClient(builder); + Assert.Throws(() => _provider.FetchDatasetByID(int.Parse(url))); + } + + private object SucessResponseString = new + { + id = "22", + name = "AddExistingDatasetWithReturnTest_Success", + type = "Dataset", + url = "22", + source = "Jira", + _links = new + { + self = "https://api.atlassian.com/jsm/assets/workspace/1234/v1/object/22", + } + }; + + private object SucessCreateString = new + { + + }; + + private List SucessObjectTypesString = [new { name = "Dataset", id = 1, objectSchemaId = 4 }]; + + private List SuccessObjectTypeAttributes = [ + new {name="Name",id=2 } + ]; + + private object SucessAQLResult = new + { + values = new List() + }; + } +} diff --git a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj index 4821571fde..c449d5feca 100644 --- a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj +++ b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj @@ -68,6 +68,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs index 5a52e9255b..11cc2c650f 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -29,8 +29,7 @@ public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicA _includeDeprecated = includeDeprecated; } - - public override void Execute() + public List GetCatalogues() { var catalogues = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); if (!_includeInternal) @@ -49,6 +48,13 @@ public override void Execute() { catalogues = catalogues.Where(c => !c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList(); } + return catalogues; + } + + + public override void Execute() + { + var catalogues = GetCatalogues(); //todo check this catalogue filtering works foreach (var catalogue in catalogues) { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs index 970be25ec6..d873a7693b 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs @@ -27,5 +27,14 @@ public override string GetRemoteID() { return Url.Split('?')[0].Split('/').Last(); } + + public string GetDOI() + { + var version = data?.versions?.FirstOrDefault(); + if (version != null) { + return version.metadata?.metadata?.summary?.doiName?.ToString(); + } + return null; + } } } diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs index 29b04a4612..36e5dd71fc 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs @@ -8,6 +8,9 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { + /// + /// + /// public class PatchSubMetadata { public object observations { get; set; } diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs index 92325b2e4d..a9311a4683 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs @@ -6,6 +6,9 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { + /// + /// + /// public class PostMetadata { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs index 12e12c0d3b..c2f4569bb5 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPatch.cs @@ -9,7 +9,9 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR { - + /// + /// + /// public class HDRDatasetPatch { public int id { get; set; } diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs index 19d1b6c5ac..6fb541c609 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -23,9 +23,12 @@ public class HDRDatasetProvider : PluginDatasetProvider { private HttpClient _client; private DatasetProviderConfiguration _configuration; + + private readonly string _inputSchema = "HDRUK"; + private readonly string _inputVersion = "3.0.0"; public HDRDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration, HttpClient client = null) : base(activator, configuration) { - _client = new HttpClient(); + _client = client??new HttpClient(); var credentials = Repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); var apiKey = credentials.GetDecryptedPassword(); _client.DefaultRequestHeaders.Add("x-application-id", credentials.Username); @@ -41,7 +44,7 @@ public override Dataset AddExistingDatasetWithReturn(string name, string url) Url = url, Type = ToString(), Provider_ID = Configuration.ID, - //DigitalObjectIdentifier = hdrDataset.data.versions.First().metadata.metadata.summary.doiName.ToString(), + DigitalObjectIdentifier = hdrDataset.GetDOI(), Folder = $"\\{Configuration.Name}", }; dataset.SaveToDatabase(); @@ -49,6 +52,7 @@ public override Dataset AddExistingDatasetWithReturn(string name, string url) return dataset; } + public override void AddExistingDataset(string name, string url) { AddExistingDatasetWithReturn(name, url); @@ -61,7 +65,7 @@ private class CreateDatasetResponse public override Dataset Create(Catalogue catalogue) { - var url = Configuration.Url + "/v1/integrations/datasets?input_schema=HDRUK&input_version=3.0.0"; + var url = Configuration.Url + $"/v1/integrations/datasets?input_schema={_inputSchema}&input_version={_inputVersion}"; var serializeOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -93,7 +97,7 @@ public override Dataset Create(Catalogue catalogue) public HDRDataset FetchHDRDataset(Dataset dataset) { - var response = Task.Run(async () => await _client.GetAsync($"{dataset.Url}?schema_model=HDRUK&schema_version=3.0.0")).Result; + var response = Task.Run(async () => await _client.GetAsync($"{dataset.Url}?schema_model={_inputSchema}&schema_version={_inputVersion}")).Result; if (response.StatusCode == HttpStatusCode.OK) { var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; @@ -107,7 +111,7 @@ public HDRDataset FetchHDRDataset(Dataset dataset) public override Dataset FetchDatasetByID(int id) { var url = Configuration.Url + "/v1/datasets/" + id; - var response = Task.Run(async () => await _client.GetAsync($"{url}?schema_model=HDRUK&schema_version=3.0.0")).Result; + var response = Task.Run(async () => await _client.GetAsync($"{url}?schema_model={_inputSchema}&schema_version={_inputVersion}")).Result; if (response.StatusCode == HttpStatusCode.OK) { var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; @@ -141,7 +145,7 @@ public override void Update(string uuid, PluginDataset datasetUpdates) }; System.Text.Json.JsonSerializer.Serialize(stream, updateObj.metadata, serializeOptions); var jsonString = "{\"metadata\":{\"metadata\":" + Encoding.UTF8.GetString(stream.ToArray()) + "}}"; - var uri = $"{Configuration.Url}/v1/integrations/datasets/{uuid}?input_schema=HDRUK&input_version=3.0.0"; + var uri = $"{Configuration.Url}/v1/integrations/datasets/{uuid}?input_schema={_inputSchema}&schema_version={_inputVersion}"; var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; @@ -259,7 +263,7 @@ public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) hdrDataset.data.versions.First().metadata.metadata.accessibility.access.dataController = catalogue.DataController; hdrDataset.data.versions.First().metadata.metadata.accessibility.access.dataProcessor = catalogue.DataProcessor; - hdrDataset.data.versions.First().metadata.metadata.identifier = "05ec5a13-3955-45a3-b449-8aba78622113";// hdrDataset.data.versions.First().metadata.identifier; + hdrDataset.data.versions.First().metadata.metadata.identifier = "FROM_RDMP";//"05ec5a13-3955-45a3-b449-8aba78622113";// hdrDataset.data.versions.First().metadata.identifier; Update(hdrDataset.data.id.ToString(), hdrDataset); } From 9dc7e01f19b164f52a4dfd172c89810a78ca0635 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 30 Apr 2025 15:29:21 +0100 Subject: [PATCH 11/20] fix class documentation --- .../Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs | 3 +++ Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs index 9772b884f9..d4d3666693 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs @@ -6,6 +6,9 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { + /// + /// + /// public class PatchMetadata { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs index a045b31b89..2d0551ca62 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs @@ -8,6 +8,9 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR { + /// + /// + /// public class HDRDatasetPost { From 1336655f36bf0262b65f183226cb983f3faa633c Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 1 May 2025 08:02:15 +0100 Subject: [PATCH 12/20] add comments for test --- Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs index d873a7693b..d3566a0475 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs @@ -8,7 +8,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR { /// - /// + /// Mapping of HDR Dataset object to C# /// public class HDRDataset : PluginDataset { From 9c5c02759ba60bfe5a4a27700dd584ed4cbf216e Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 1 May 2025 10:49:02 +0100 Subject: [PATCH 13/20] fix up --- Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs | 2 +- Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs index 2d0551ca62..c30fa0cdb5 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetPost.cs @@ -22,7 +22,7 @@ public HDRDatasetPost(Catalogue catalogue) { metadata.modified = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); metadata.issued = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); metadata.summary.title=catalogue.Name; - metadata.summary.@abstract = catalogue.ShortDescription.Length>4?catalogue.ShortDescription:""; + metadata.summary.@abstract = catalogue.ShortDescription!= null && catalogue.ShortDescription.Length>4?catalogue.ShortDescription:""; metadata.summary.dataCustodian = new DataCustodian(); metadata.summary.dataCustodian.identifier= "unknown"; metadata.summary.dataCustodian.name = "name"; diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs index 6fb541c609..050ee479bd 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -84,6 +84,7 @@ public override Dataset Create(Catalogue catalogue) var content = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; var responseJson = JsonConvert.DeserializeObject(content); var dataset = FetchDatasetByID(responseJson.data) as HDRDataset; + dataset.Url = dataset.data.id.ToString(); UpdateUsingCatalogue(dataset, catalogue); return dataset; } @@ -239,7 +240,7 @@ public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) { var hdrDataset = (HDRDataset)FetchDatasetByID(int.Parse(dataset.Url)); hdrDataset.data.versions.First().metadata.metadata.summary.title = catalogue.Name; - hdrDataset.data.versions.First().metadata.metadata.summary.@abstract = catalogue.ShortDescription.Length <5? catalogue.ShortDescription.PadRight(5): catalogue.ShortDescription; + hdrDataset.data.versions.First().metadata.metadata.summary.@abstract = catalogue.ShortDescription != null && catalogue.ShortDescription.Length <5? catalogue.ShortDescription.PadRight(5): catalogue.ShortDescription; hdrDataset.data.versions.First().metadata.metadata.summary.contactPoint = catalogue.Administrative_contact_email; hdrDataset.data.versions.First().metadata.metadata.summary.keywords = (catalogue.Search_keywords ?? "").Split(',').Cast().Where(k => k != "").Cast().ToList(); hdrDataset.data.versions.First().metadata.metadata.summary.doiName = catalogue.Doi; @@ -263,7 +264,7 @@ public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) hdrDataset.data.versions.First().metadata.metadata.accessibility.access.dataController = catalogue.DataController; hdrDataset.data.versions.First().metadata.metadata.accessibility.access.dataProcessor = catalogue.DataProcessor; - hdrDataset.data.versions.First().metadata.metadata.identifier = "FROM_RDMP";//"05ec5a13-3955-45a3-b449-8aba78622113";// hdrDataset.data.versions.First().metadata.identifier; + hdrDataset.data.versions.First().metadata.metadata.identifier = "05ec5a13-3955-45a3-b449-8aba78622113";// hdrDataset.data.versions.First().metadata.identifier; Update(hdrDataset.data.id.ToString(), hdrDataset); } From f0ca2808cd73bd89426c71000d74f28ab903faff Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 1 May 2025 13:06:37 +0100 Subject: [PATCH 14/20] fix update --- .../Data/Datasets/HDR/HDRDatasetProvider.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs index 050ee479bd..cf86ef1fc4 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -28,7 +28,7 @@ public class HDRDatasetProvider : PluginDatasetProvider private readonly string _inputVersion = "3.0.0"; public HDRDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration, HttpClient client = null) : base(activator, configuration) { - _client = client??new HttpClient(); + _client = client ?? new HttpClient(); var credentials = Repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); var apiKey = credentials.GetDecryptedPassword(); _client.DefaultRequestHeaders.Add("x-application-id", credentials.Username); @@ -145,10 +145,10 @@ public override void Update(string uuid, PluginDataset datasetUpdates) metadata = new HDRDatasetPatch((HDRDataset)datasetUpdates).metadata.metadata }; System.Text.Json.JsonSerializer.Serialize(stream, updateObj.metadata, serializeOptions); - var jsonString = "{\"metadata\":{\"metadata\":" + Encoding.UTF8.GetString(stream.ToArray()) + "}}"; - var uri = $"{Configuration.Url}/v1/integrations/datasets/{uuid}?input_schema={_inputSchema}&schema_version={_inputVersion}"; - var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + var jsonString = "{\"metadata\":" + Encoding.UTF8.GetString(stream.ToArray()) + "}"; + var uri = $"{Configuration.Url}/v1/integrations/datasets/{uuid}?input_schema={_inputSchema}&input_version={_inputVersion}"; + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; if (response.StatusCode != HttpStatusCode.OK) { @@ -240,7 +240,7 @@ public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) { var hdrDataset = (HDRDataset)FetchDatasetByID(int.Parse(dataset.Url)); hdrDataset.data.versions.First().metadata.metadata.summary.title = catalogue.Name; - hdrDataset.data.versions.First().metadata.metadata.summary.@abstract = catalogue.ShortDescription != null && catalogue.ShortDescription.Length <5? catalogue.ShortDescription.PadRight(5): catalogue.ShortDescription; + hdrDataset.data.versions.First().metadata.metadata.summary.@abstract = catalogue.ShortDescription != null && catalogue.ShortDescription.Length < 5 ? catalogue.ShortDescription.PadRight(5) : catalogue.ShortDescription; hdrDataset.data.versions.First().metadata.metadata.summary.contactPoint = catalogue.Administrative_contact_email; hdrDataset.data.versions.First().metadata.metadata.summary.keywords = (catalogue.Search_keywords ?? "").Split(',').Cast().Where(k => k != "").Cast().ToList(); hdrDataset.data.versions.First().metadata.metadata.summary.doiName = catalogue.Doi; @@ -250,7 +250,7 @@ public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) hdrDataset.data.versions.First().metadata.metadata.coverage.spatial = catalogue.Geographical_coverage; - if(catalogue.DataType != null) + if (catalogue.DataType != null) hdrDataset.data.versions.First().metadata.metadata.provenance.origin.datasetType = catalogue.DataType.Split(",").Select(MapDataTypeToHDRDataType).ToList(); if (catalogue.DataSubType != null) hdrDataset.data.versions.First().metadata.metadata.provenance.origin.datasetSubType = catalogue.DataSubType.Split(",").Select(MapDataSubTypeToHDR).ToList(); @@ -266,6 +266,7 @@ public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) hdrDataset.data.versions.First().metadata.metadata.identifier = "05ec5a13-3955-45a3-b449-8aba78622113";// hdrDataset.data.versions.First().metadata.identifier; + Update(hdrDataset.data.id.ToString(), hdrDataset); } From a8f9d0aa9d45659dd06980d9274da091b3f6a1e5 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 6 May 2025 09:24:42 +0100 Subject: [PATCH 15/20] tidy up --- .../Data/Datasets/HDR/HDRDatasetItems/Access.cs | 2 +- .../HDR/HDRDatasetItems/Accessibility.cs | 2 +- .../Datasets/HDR/HDRDatasetItems/Coverage.cs | 2 +- .../Data/Datasets/HDR/HDRDatasetItems/Data.cs | 2 +- .../HDR/HDRDatasetItems/DataCustodian.cs | 2 +- .../HDR/HDRDatasetItems/Documentation.cs | 2 +- .../HDR/HDRDatasetItems/EnrichmentAndLinkage.cs | 2 +- .../HDR/HDRDatasetItems/FormatAndStandards.cs | 2 +- .../Datasets/HDR/HDRDatasetItems/Metadata.cs | 2 +- .../Datasets/HDR/HDRDatasetItems/NamedEntity.cs | 2 +- .../Data/Datasets/HDR/HDRDatasetItems/Origin.cs | 2 +- .../HDR/HDRDatasetItems/PatchMetadata.cs | 2 +- .../HDR/HDRDatasetItems/PatchSubMetadata.cs | 2 +- .../Datasets/HDR/HDRDatasetItems/PostMetadata.cs | 2 +- .../Datasets/HDR/HDRDatasetItems/Provenance.cs | 2 +- .../Datasets/HDR/HDRDatasetItems/Revision.cs | 2 +- .../HDR/HDRDatasetItems/SpatialCoverage.cs | 2 +- .../HDR/HDRDatasetItems/StructuralMetadata.cs | 2 +- .../Data/Datasets/HDR/HDRDatasetItems/Summary.cs | 2 +- .../Data/Datasets/HDR/HDRDatasetItems/Team.cs | 2 +- .../Datasets/HDR/HDRDatasetItems/Temporal.cs | 2 +- .../Data/Datasets/HDR/HDRDatasetItems/Usage.cs | 2 +- .../Data/Datasets/HDR/HDRDatasetItems/Version.cs | 2 +- .../Data/Datasets/HDR/HDRDatasetProvider.cs | 16 +++++++++------- 24 files changed, 32 insertions(+), 30 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Access.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Access.cs index a8cf4fa7fb..3db5439f25 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Access.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Access.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Access { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Accessibility.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Accessibility.cs index 6e657152c7..0119fec913 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Accessibility.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Accessibility.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Accessibility { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Coverage.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Coverage.cs index ef1678c629..093f485f57 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Coverage.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Coverage.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Coverage { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Data.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Data.cs index fb482dd0df..87a99665bc 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Data.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Data.cs @@ -8,7 +8,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Data { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/DataCustodian.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/DataCustodian.cs index b229b14ed1..c6932cfb94 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/DataCustodian.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/DataCustodian.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class DataCustodian { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Documentation.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Documentation.cs index 177edbdd41..86a72487ce 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Documentation.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Documentation.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Documentation { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/EnrichmentAndLinkage.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/EnrichmentAndLinkage.cs index 69583497da..268afd3bfa 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/EnrichmentAndLinkage.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/EnrichmentAndLinkage.cs @@ -8,7 +8,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class EnrichmentAndLinkage { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/FormatAndStandards.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/FormatAndStandards.cs index 334ec7af94..1538ff0e39 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/FormatAndStandards.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/FormatAndStandards.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class FormatAndStandards { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Metadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Metadata.cs index d17e508565..0efa87d688 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Metadata.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Metadata.cs @@ -10,7 +10,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Metadata { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/NamedEntity.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/NamedEntity.cs index 4674d546a3..1767847fb5 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/NamedEntity.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/NamedEntity.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class NamedEntity { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Origin.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Origin.cs index c281dcd93a..5e2b7a2685 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Origin.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Origin.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Origin { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs index d4d3666693..1a0fe0021b 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchMetadata.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Used to map patch updates to HDR API /// public class PatchMetadata { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs index 36e5dd71fc..4b12decf6f 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PatchSubMetadata.cs @@ -9,7 +9,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class PatchSubMetadata { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs index a9311a4683..8127d09c30 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/PostMetadata.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Used to map create to HDR API /// public class PostMetadata { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Provenance.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Provenance.cs index a72b4a4cb9..3916bdad1e 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Provenance.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Provenance.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Provenance { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Revision.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Revision.cs index 22236ed4bd..e1483c3bbe 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Revision.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Revision.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Revision { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/SpatialCoverage.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/SpatialCoverage.cs index 635b0f9ee8..de28cfb1dc 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/SpatialCoverage.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/SpatialCoverage.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class SpatialCoverage { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/StructuralMetadata.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/StructuralMetadata.cs index 17cfaa55fc..1d3903d808 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/StructuralMetadata.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/StructuralMetadata.cs @@ -8,7 +8,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class StructuralMetadata { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Summary.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Summary.cs index fdb64724c1..d6999e1142 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Summary.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Summary.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Summary { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Team.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Team.cs index 9837ac6d29..3562a8ea94 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Team.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Team.cs @@ -9,7 +9,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Team { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Temporal.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Temporal.cs index d0718ce672..2d5a43aaa4 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Temporal.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Temporal.cs @@ -9,7 +9,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Temporal { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Usage.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Usage.cs index 0739c79082..d9d140c62f 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Usage.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Usage.cs @@ -7,7 +7,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Usage { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Version.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Version.cs index 0d00627d2a..89c555d8ef 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Version.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Version.cs @@ -8,7 +8,7 @@ namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems { /// - /// + /// Mapping from HDR API /// public class Version { diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs index cf86ef1fc4..07d7d6952f 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -23,6 +23,7 @@ public class HDRDatasetProvider : PluginDatasetProvider { private HttpClient _client; private DatasetProviderConfiguration _configuration; + private readonly string _url; private readonly string _inputSchema = "HDRUK"; private readonly string _inputVersion = "3.0.0"; @@ -34,6 +35,7 @@ public HDRDatasetProvider(IBasicActivateItems activator, DatasetProviderConfigur _client.DefaultRequestHeaders.Add("x-application-id", credentials.Username); _client.DefaultRequestHeaders.Add("x-client-id", apiKey); _configuration = configuration; + _url = Configuration.Url.TrimEnd('/'); } public override Dataset AddExistingDatasetWithReturn(string name, string url) { @@ -45,7 +47,7 @@ public override Dataset AddExistingDatasetWithReturn(string name, string url) Type = ToString(), Provider_ID = Configuration.ID, DigitalObjectIdentifier = hdrDataset.GetDOI(), - Folder = $"\\{Configuration.Name}", + Folder = $"\\HDR\\{Configuration.Name}", }; dataset.SaveToDatabase(); Activator.Publish(dataset); @@ -65,7 +67,7 @@ private class CreateDatasetResponse public override Dataset Create(Catalogue catalogue) { - var url = Configuration.Url + $"/v1/integrations/datasets?input_schema={_inputSchema}&input_version={_inputVersion}"; + var url = _url + $"/v1/integrations/datasets?input_schema={_inputSchema}&input_version={_inputVersion}"; var serializeOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -111,7 +113,7 @@ public HDRDataset FetchHDRDataset(Dataset dataset) public override Dataset FetchDatasetByID(int id) { - var url = Configuration.Url + "/v1/datasets/" + id; + var url = _url + "/v1/datasets/" + id; var response = Task.Run(async () => await _client.GetAsync($"{url}?schema_model={_inputSchema}&schema_version={_inputVersion}")).Result; if (response.StatusCode == HttpStatusCode.OK) { @@ -147,13 +149,13 @@ public override void Update(string uuid, PluginDataset datasetUpdates) System.Text.Json.JsonSerializer.Serialize(stream, updateObj.metadata, serializeOptions); var jsonString = "{\"metadata\":" + Encoding.UTF8.GetString(stream.ToArray()) + "}"; - var uri = $"{Configuration.Url}/v1/integrations/datasets/{uuid}?input_schema={_inputSchema}&input_version={_inputVersion}"; + var uri = $"{_url}/v1/integrations/datasets/{uuid}?input_schema={_inputSchema}&input_version={_inputVersion}"; var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; if (response.StatusCode != HttpStatusCode.OK) { - var x = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - throw new Exception(x); + var errorMessage = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + throw new Exception(errorMessage); } } @@ -264,7 +266,7 @@ public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) hdrDataset.data.versions.First().metadata.metadata.accessibility.access.dataController = catalogue.DataController; hdrDataset.data.versions.First().metadata.metadata.accessibility.access.dataProcessor = catalogue.DataProcessor; - hdrDataset.data.versions.First().metadata.metadata.identifier = "05ec5a13-3955-45a3-b449-8aba78622113";// hdrDataset.data.versions.First().metadata.identifier; + hdrDataset.data.versions.First().metadata.metadata.identifier = Guid.NewGuid().ToString(); Update(hdrDataset.data.id.ToString(), hdrDataset); From 281b0d6066d676265a87f5876014392ed65be6d1 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 6 May 2025 10:55:26 +0100 Subject: [PATCH 16/20] add get url --- Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs | 5 +++++ Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs index 07d7d6952f..b3957cc1da 100644 --- a/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetProvider.cs @@ -272,6 +272,11 @@ public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) Update(hdrDataset.data.id.ToString(), hdrDataset); } + public override string GetRemoteURL(Dataset dataset) + { + return $"{_url.Replace("api.", "web.").Replace("/api", "/en")}/dataset/{dataset.Url}"; + } + private class HDRUpdateObject { public int team_id { get; set; } diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs index c6a952ad06..d893742af3 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDatasetProvider.cs @@ -40,7 +40,7 @@ public virtual void AddExistingDataset(string name, string url) throw new NotImplementedException(); } - public string GetRemoteURL(Dataset dataset) + public virtual string GetRemoteURL(Dataset dataset) { throw new NotImplementedException(); } From 462a40bf015ee3c068f9e4a8f2e148939cb2cc08 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 09:30:23 +0100 Subject: [PATCH 17/20] update version --- SharedAssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 559a9fe198..6096b7fbb7 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: AssemblyVersion("9.0.0")] [assembly: AssemblyFileVersion("9.0.0")] -[assembly: AssemblyInformationalVersion("9.0.0")] +[assembly: AssemblyInformationalVersion("9.0.0-right-click-test")] From d7a793260b9f54f89f5a8045e73ae1787325b8d0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 09:31:04 +0100 Subject: [PATCH 18/20] update version --- SharedAssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 6096b7fbb7..559a9fe198 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: AssemblyVersion("9.0.0")] [assembly: AssemblyFileVersion("9.0.0")] -[assembly: AssemblyInformationalVersion("9.0.0-right-click-test")] +[assembly: AssemblyInformationalVersion("9.0.0")] From 360e4f650a6f1e3c53cffb723b812e58733279b0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 10:21:17 +0100 Subject: [PATCH 19/20] fix merge --- ...ngCataloguesIntoExternalDatasetProvider.cs | 19 ++++++++++--------- Tests.Common/UnitTests.cs | 1 + 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs index a3a572722d..e8d28acb25 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -33,24 +33,25 @@ public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicA public List GetCatalogues() { + var cataloguesToReturn = new List(); var catalogues = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects().ToList(); - if (!_includeInternal) + if (_includeInternal) { - catalogues = catalogues.Where(c => !c.IsInternalDataset).ToList(); + cataloguesToReturn.AddRange(catalogues.Where(c => c.IsInternalDataset).ToList()); } - if (!_includeProjectSpecific) + if (_includeProjectSpecific) { - catalogues = catalogues.Where(c => !c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)).ToList(); + cataloguesToReturn.AddRange(catalogues.Where(c => c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)).ToList()); } - if (!_includeDeprecated) + if (_includeDeprecated) { - catalogues = catalogues.Where(c => !c.IsDeprecated).ToList(); + cataloguesToReturn.AddRange(catalogues.Where(c => c.IsDeprecated).ToList()); } - if (!_includeExtractable) + if (_includeExtractable) { - catalogues = catalogues.Where(c => !c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList(); + cataloguesToReturn.AddRange(catalogues.Where(c => c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList()); } - return catalogues; + return cataloguesToReturn.Distinct().ToList(); } diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index a591106dec..ce9c1a8aa3 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -626,6 +626,7 @@ public static T WhenIHaveA(MemoryDataExportRepository repository) where T : D if (typeof(T) == typeof(HDRDataset)) { return (T)(object)new HDRDataset(repository.CatalogueRepository, "HDR Dataset"); + } if(typeof(T) == typeof(ExtractableDataSetProject)) { return (T)(object)new ExtractableDataSetProject(repository, WhenIHaveA(repository), WhenIHaveA(repository)); From e3fe6a595d2cd70f09a05bef279d6e4fcb8c7273 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 28 Jul 2025 10:51:57 +0100 Subject: [PATCH 20/20] improve ui --- .../MainFormUITabs/CatalogueUI.Designer.cs | 160 +++++++++--------- Rdmp.UI/MainFormUITabs/CatalogueUI.cs | 21 ++- 2 files changed, 94 insertions(+), 87 deletions(-) diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs index ff2a867742..6a807d653c 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.Designer.cs @@ -36,98 +36,99 @@ private void InitializeComponent() cbColdStorage = new CheckBox(); cbInternal = new CheckBox(); cbDeprecated = new CheckBox(); - editableFolder = new SimpleControls.EditableLabelUI(); - editableCatalogueName = new SimpleControls.EditableLabelUI(); + editableFolder = new Rdmp.UI.SimpleControls.EditableLabelUI(); + editableCatalogueName = new Rdmp.UI.SimpleControls.EditableLabelUI(); ticketingControl1 = new TicketingControlUI(); tabControl1 = new TabControl(); tabPage1 = new TabPage(); groupBox23 = new GroupBox(); - aiAcronym = new SimpleControls.AdditionalInfomationUI(); + aiAcronym = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); tbAcronym = new TextBox(); groupBox16 = new GroupBox(); - aiDescription = new SimpleControls.AdditionalInfomationUI(); + aiDescription = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); tbDescription = new TextBox(); groupBox15 = new GroupBox(); - aiShortDescription = new SimpleControls.AdditionalInfomationUI(); + aiShortDescription = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); tbAbstract = new TextBox(); tabPage2 = new TabPage(); tableLayoutPanel2 = new TableLayoutPanel(); groupBox18 = new GroupBox(); - aiDatasetType = new SimpleControls.AdditionalInfomationUI(); - ddDatasetType = new SimpleControls.MultiSelectChips.DropdownOptionsChipDisplay(); + aiDatasetType = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + ddDatasetType = new Rdmp.UI.SimpleControls.MultiSelectChips.DropdownOptionsChipDisplay(); groupBox22 = new GroupBox(); - aiKeywords = new SimpleControls.AdditionalInfomationUI(); - ffcKeywords = new SimpleControls.MultiSelectChips.FreeFormTextChipDisplay(); + aiKeywords = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + ffcKeywords = new Rdmp.UI.SimpleControls.MultiSelectChips.FreeFormTextChipDisplay(); groupBox19 = new GroupBox(); - aiDatasetSubtype = new SimpleControls.AdditionalInfomationUI(); - ddDatasetSubtype = new SimpleControls.MultiSelectChips.DropdownOptionsChipDisplay(); + aiDatasetSubtype = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + ddDatasetSubtype = new Rdmp.UI.SimpleControls.MultiSelectChips.DropdownOptionsChipDisplay(); groupBox21 = new GroupBox(); - ddDataSourceSetting = new SimpleControls.MultiSelectChips.DropdownOptionsChipDisplay(); - aiDataSourceSetting = new SimpleControls.AdditionalInfomationUI(); + ddDataSourceSetting = new Rdmp.UI.SimpleControls.MultiSelectChips.DropdownOptionsChipDisplay(); + aiDataSourceSetting = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); groupBox20 = new GroupBox(); - ddDataSource = new SimpleControls.MultiSelectChips.DropdownOptionsChipDisplay(); - aiDataSource = new SimpleControls.AdditionalInfomationUI(); - groupBox24 = new GroupBox(); - aiPurposeOfDataset = new SimpleControls.AdditionalInfomationUI(); - cbPurpose = new ComboBox(); + ddDataSource = new Rdmp.UI.SimpleControls.MultiSelectChips.DropdownOptionsChipDisplay(); + aiDataSource = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); groupBox17 = new GroupBox(); - aiResourceType = new SimpleControls.AdditionalInfomationUI(); + aiResourceType = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); cb_resourceType = new ComboBox(); + groupBox24 = new GroupBox(); + aiPurposeOfDataset = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + cbPurpose = new ComboBox(); tabPage3 = new TabPage(); groupBox14 = new GroupBox(); - aiEndDate = new SimpleControls.AdditionalInfomationUI(); + aiEndDate = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); dtpEndDate = new DateTimePicker(); btnEndDateClear = new Button(); groupBox13 = new GroupBox(); - aiStartDate = new SimpleControls.AdditionalInfomationUI(); + aiStartDate = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); dtpStart = new DateTimePicker(); btnStartDateClear = new Button(); groupBox12 = new GroupBox(); - aiGranularity = new SimpleControls.AdditionalInfomationUI(); + aiGranularity = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); cb_granularity = new ComboBox(); groupBox11 = new GroupBox(); - aiGeographicalCoverage = new SimpleControls.AdditionalInfomationUI(); + aiGeographicalCoverage = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); tbGeoCoverage = new TextBox(); tabPage4 = new TabPage(); groupBox10 = new GroupBox(); - aiJuristiction = new SimpleControls.AdditionalInfomationUI(); + aiJuristiction = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); tbJuristiction = new TextBox(); groupBox9 = new GroupBox(); - aiDataProcessor = new SimpleControls.AdditionalInfomationUI(); + aiDataProcessor = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); tbDataProcessor = new TextBox(); groupBox8 = new GroupBox(); - aiDataController = new SimpleControls.AdditionalInfomationUI(); + aiDataController = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); tbDataController = new TextBox(); groupBox7 = new GroupBox(); - aiAccessContact = new SimpleControls.AdditionalInfomationUI(); + aiAccessContact = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); tbAccessContact = new TextBox(); tabPage5 = new TabPage(); groupBox3 = new GroupBox(); - aiDOI = new SimpleControls.AdditionalInfomationUI(); + aiDOI = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); tbDOI = new TextBox(); tableLayoutPanel1 = new TableLayoutPanel(); groupBox2 = new GroupBox(); - aiControlledGroup = new SimpleControls.AdditionalInfomationUI(); - fftControlledVocab = new SimpleControls.MultiSelectChips.FreeFormTextChipDisplay(); + aiControlledGroup = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + fftControlledVocab = new Rdmp.UI.SimpleControls.MultiSelectChips.FreeFormTextChipDisplay(); groupBox1 = new GroupBox(); - aiPeople = new SimpleControls.AdditionalInfomationUI(); - ffcPeople = new SimpleControls.MultiSelectChips.FreeFormTextChipDisplay(); + aiPeople = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + ffcPeople = new Rdmp.UI.SimpleControls.MultiSelectChips.FreeFormTextChipDisplay(); tabPage6 = new TabPage(); - aiUpdateFrequency = new SimpleControls.AdditionalInfomationUI(); + aiUpdateFrequency = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); groupBox6 = new GroupBox(); - aiUpdateLag = new SimpleControls.AdditionalInfomationUI(); + aiUpdateLag = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); cbUpdateLag = new ComboBox(); groupBox5 = new GroupBox(); - aiInitialReleaseDate = new SimpleControls.AdditionalInfomationUI(); + aiInitialReleaseDate = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); dtpReleaseDate = new DateTimePicker(); btnReleaseDateClear = new Button(); groupBox4 = new GroupBox(); cb_updateFrequency = new ComboBox(); tabPage7 = new TabPage(); groupBox25 = new GroupBox(); - aiAssociatedMedia = new SimpleControls.AdditionalInfomationUI(); - ffAssociatedMedia = new SimpleControls.MultiSelectChips.FreeFormTextChipDisplay(); + aiAssociatedMedia = new Rdmp.UI.SimpleControls.AdditionalInfomationUI(); + ffAssociatedMedia = new Rdmp.UI.SimpleControls.MultiSelectChips.FreeFormTextChipDisplay(); tabPage8 = new TabPage(); + button1 = new Button(); tableLayoutPanel3 = new TableLayoutPanel(); label6 = new Label(); label4 = new Label(); @@ -136,7 +137,6 @@ private void InitializeComponent() label2 = new Label(); label5 = new Label(); button2 = new Button(); - button1 = new Button(); ((System.ComponentModel.ISupportInitialize)splitContainer1).BeginInit(); splitContainer1.Panel1.SuspendLayout(); splitContainer1.Panel2.SuspendLayout(); @@ -153,8 +153,8 @@ private void InitializeComponent() groupBox19.SuspendLayout(); groupBox21.SuspendLayout(); groupBox20.SuspendLayout(); - groupBox24.SuspendLayout(); groupBox17.SuspendLayout(); + groupBox24.SuspendLayout(); tabPage3.SuspendLayout(); groupBox14.SuspendLayout(); groupBox13.SuspendLayout(); @@ -268,7 +268,6 @@ private void InitializeComponent() tabControl1.Controls.Add(tabPage6); tabControl1.Controls.Add(tabPage7); tabControl1.Controls.Add(tabPage8); - tabControl1.Dock = DockStyle.Fill; tabControl1.Location = new System.Drawing.Point(0, 0); tabControl1.Name = "tabControl1"; @@ -555,33 +554,6 @@ private void InitializeComponent() aiDataSource.Size = new System.Drawing.Size(20, 20); aiDataSource.TabIndex = 24; // - // groupBox24 - // - groupBox24.AutoSize = true; - groupBox24.Controls.Add(aiPurposeOfDataset); - groupBox24.Controls.Add(cbPurpose); - groupBox24.Location = new System.Drawing.Point(207, 6); - groupBox24.Name = "groupBox24"; - groupBox24.Size = new System.Drawing.Size(194, 67); - groupBox24.TabIndex = 20; - groupBox24.TabStop = false; - groupBox24.Text = "Purpose of Dataset"; - // - // aiPurposeOfDataset - // - aiPurposeOfDataset.Location = new System.Drawing.Point(114, 0); - aiPurposeOfDataset.Name = "aiPurposeOfDataset"; - aiPurposeOfDataset.Size = new System.Drawing.Size(20, 20); - aiPurposeOfDataset.TabIndex = 27; - // - // cbPurpose - // - cbPurpose.FormattingEnabled = true; - cbPurpose.Location = new System.Drawing.Point(6, 22); - cbPurpose.Name = "cbPurpose"; - cbPurpose.Size = new System.Drawing.Size(182, 23); - cbPurpose.TabIndex = 1; - // // groupBox17 // groupBox17.AutoSize = true; @@ -609,6 +581,33 @@ private void InitializeComponent() cb_resourceType.Size = new System.Drawing.Size(182, 23); cb_resourceType.TabIndex = 0; // + // groupBox24 + // + groupBox24.AutoSize = true; + groupBox24.Controls.Add(aiPurposeOfDataset); + groupBox24.Controls.Add(cbPurpose); + groupBox24.Location = new System.Drawing.Point(207, 6); + groupBox24.Name = "groupBox24"; + groupBox24.Size = new System.Drawing.Size(194, 67); + groupBox24.TabIndex = 20; + groupBox24.TabStop = false; + groupBox24.Text = "Purpose of Dataset"; + // + // aiPurposeOfDataset + // + aiPurposeOfDataset.Location = new System.Drawing.Point(114, 0); + aiPurposeOfDataset.Name = "aiPurposeOfDataset"; + aiPurposeOfDataset.Size = new System.Drawing.Size(20, 20); + aiPurposeOfDataset.TabIndex = 27; + // + // cbPurpose + // + cbPurpose.FormattingEnabled = true; + cbPurpose.Location = new System.Drawing.Point(6, 22); + cbPurpose.Name = "cbPurpose"; + cbPurpose.Size = new System.Drawing.Size(182, 23); + cbPurpose.TabIndex = 1; + // // tabPage3 // tabPage3.AutoScroll = true; @@ -1126,6 +1125,16 @@ private void InitializeComponent() tabPage8.TabIndex = 7; tabPage8.Text = "Datasets"; // + // button1 + // + button1.Location = new System.Drawing.Point(163, 12); + button1.Name = "button1"; + button1.Size = new System.Drawing.Size(183, 23); + button1.TabIndex = 3; + button1.Text = "Create using External Provider"; + button1.UseVisualStyleBackColor = true; + button1.Click += button1_Click; + // // tableLayoutPanel3 // tableLayoutPanel3.AutoSize = true; @@ -1137,12 +1146,12 @@ private void InitializeComponent() tableLayoutPanel3.ColumnStyles.Add(new ColumnStyle()); tableLayoutPanel3.ColumnStyles.Add(new ColumnStyle()); tableLayoutPanel3.ColumnStyles.Add(new ColumnStyle()); - tableLayoutPanel3.Controls.Add(label6, 5, 0); - tableLayoutPanel3.Controls.Add(label4, 3, 0); - tableLayoutPanel3.Controls.Add(label3, 2, 0); tableLayoutPanel3.Controls.Add(label1, 0, 0); tableLayoutPanel3.Controls.Add(label2, 1, 0); + tableLayoutPanel3.Controls.Add(label3, 2, 0); + tableLayoutPanel3.Controls.Add(label4, 3, 0); tableLayoutPanel3.Controls.Add(label5, 4, 0); + tableLayoutPanel3.Controls.Add(label6, 5, 0); tableLayoutPanel3.Location = new System.Drawing.Point(7, 41); tableLayoutPanel3.Name = "tableLayoutPanel3"; tableLayoutPanel3.RowCount = 2; @@ -1217,15 +1226,6 @@ private void InitializeComponent() button2.UseVisualStyleBackColor = true; button2.Click += button2_Click; // - // button1 - // - button1.Location = new System.Drawing.Point(163, 12); - button1.Name = "button1"; - button1.Size = new System.Drawing.Size(183, 23); - button1.TabIndex = 3; - button1.Text = "Create using External Provider"; - button1.UseVisualStyleBackColor = true; - button1.Click += button1_Click; // CatalogueUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); @@ -1261,8 +1261,8 @@ private void InitializeComponent() groupBox21.PerformLayout(); groupBox20.ResumeLayout(false); groupBox20.PerformLayout(); - groupBox24.ResumeLayout(false); groupBox17.ResumeLayout(false); + groupBox24.ResumeLayout(false); tabPage3.ResumeLayout(false); groupBox14.ResumeLayout(false); groupBox13.ResumeLayout(false); diff --git a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs index 2921c0bbf9..d6093b9463 100644 --- a/Rdmp.UI/MainFormUITabs/CatalogueUI.cs +++ b/Rdmp.UI/MainFormUITabs/CatalogueUI.cs @@ -460,9 +460,9 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) case 7: tableLayoutPanel3.SuspendLayout(); var datasets = _catalogue.GetLinkedDatasets(); - while (tableLayoutPanel3.Controls.Count > 0) + while (tableLayoutPanel3.Controls.Count > 6) { - tableLayoutPanel3.Controls[0].Dispose(); + tableLayoutPanel3.Controls[7].Dispose(); } foreach (var dataset in datasets) { @@ -471,7 +471,7 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) var label = new Label(); label.Text = dataset.Name; label.AutoSize = true; - tableLayoutPanel3.Controls.Add(label, 0, tableLayoutPanel3.RowCount - 1); + tableLayoutPanel3.Controls.Add(label); var cb = new CheckBox(); cb.Checked = linkage.Autoupdate; cb.CheckedChanged += (object sender, EventArgs e) => @@ -480,11 +480,18 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) linkage.SaveToDatabase(); Publish(linkage); }; - tableLayoutPanel3.Controls.Add(cb, 1, tableLayoutPanel3.RowCount - 1); + tableLayoutPanel3.Controls.Add(cb); var btn = new Button(); btn.Text = "Update Now"; btn.Click += (object sender, EventArgs e) => UpdateDataset(dataset); - tableLayoutPanel3.Controls.Add(btn, 2, tableLayoutPanel3.RowCount - 1); + tableLayoutPanel3.Controls.Add(btn); + + var viewBtn = new Button(); + viewBtn.Text = "View"; + viewBtn.Enabled = false; + tableLayoutPanel3.Controls.Add(viewBtn); + + var btn3 = new Button(); btn3.Text = "Remove Link to Dataset"; btn3.Click += (object sender, EventArgs e) => @@ -496,11 +503,11 @@ private void tabControl1_SelectedIndexChanged(object sender, EventArgs e) tabControl1_SelectedIndexChanged(tabControl1, null); } }; - tableLayoutPanel3.Controls.Add(btn3, 4, tableLayoutPanel3.RowCount - 1); + tableLayoutPanel3.Controls.Add(btn3); var labelType = new Label(); labelType.Text = _catalogue.CatalogueRepository.GetObjectByID((int)dataset.Provider_ID).Name; labelType.AutoSize = true; - tableLayoutPanel3.Controls.Add(labelType, 5, tableLayoutPanel3.RowCount - 1); + tableLayoutPanel3.Controls.Add(labelType); } tableLayoutPanel3.ResumeLayout();