From 2c03c096cdd51a505ab4bcb7a4f109c9aaa38db1 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 9 Dec 2025 13:04:15 +0000 Subject: [PATCH 01/13] add basic command --- ...cuteCommandExportCataloguesToHDRGateway.cs | 268 ++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs new file mode 100644 index 0000000000..c425e1c422 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs @@ -0,0 +1,268 @@ +using Amazon.Runtime.Internal.Endpoints.StandardLibrary; +using Amazon.SecurityToken.Model; +using Azure; +using MathNet.Numerics; +using Newtonsoft.Json; +using NPOI.POIFS.Crypt; +using NPOI.SS.UserModel.Charts; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Dataset.Confluence; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace Rdmp.Core.CommandExecution.AtomicCommands +{ + /// + /// Uses provided HDR Gateway credentials to populate the HDR gateway + /// + public class ExecuteCommandExportCataloguesToHDRGateway : BasicCommandExecution, IAtomicCommand + { + + private readonly IBasicActivateItems _activator; + private readonly string _url; + private readonly int _team; + private readonly string _appID; + private readonly string _clientID; + private readonly HttpClient _client = new(); + private readonly string _datasetsEndpoint = "/v2/datasets"; + + public class HDRDatasetMetadataSummary + { + public HDRDatasetMetadataSummary(Catalogue catalogue) + { + title = catalogue.Name; + populationSize = 0; + contactPoint = ""; + } + public HDRDatasetMetadataSummary() { } + public string @abstract { get; set; } + public string contactPoint { get; set; } + public List keywords { get; set; } + public object doiName { get; set; } + public string title { get; set; } + //public DataCustodian dataCustodian { get; set; } + public int populationSize { get; set; } + public object alternateIdentifiers { get; set; } + } + public class HDRDatasetAccess + { + public string deliveryLeadTime { get; set; } + public List jurisdiction { get; set; } + public string dataController { get; set; } + public string dataProcessor { get; set; } + public string accessRights { get; set; } + public string accessService { get; set; } + public string accessRequestCost { get; set; } + public object accessServiceCategory { get; set; } + } + + private class CustomDateTimeConverterThreeMilliseconds : JsonConverter + { + public override DateTime ReadJson(JsonReader reader, Type objectType, DateTime existingValue, bool hasExistingValue, Newtonsoft.Json.JsonSerializer serializer) + { + return DateTime.Parse(reader.ReadAsString()); + } + + + public override void WriteJson(JsonWriter writer, DateTime value, Newtonsoft.Json.JsonSerializer serializer) + { + writer.WriteValue(value.ToString("yyyy-MM-ddTHH:mm:ss.000Z", CultureInfo.InvariantCulture)); + } + } + + public class HDRDatasetTemporal + { + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + + public DateTime? endDate { get; set; } + + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + public DateTime startDate { get; set; } + public string timeLag { get; set; } + public string publishingFrequency { get; set; } + public object distributionReleaseDate { get; set; } + } + + public class HDRDatasetFormatAndStandards + { + public List conformsTo { get; set; } + public List vocabularyEncodingScheme { get; set; } + public List language { get; set; } + public List format { get; set; } + } + + public class HDRDatasetOrigin + { + public List purpose { get; set; } + public List source { get; set; } + public List collectionSource { get; set; } + public List datasetType { get; set; } + public List datasetSubType { get; set; } + public string imageContrast { get; set; } + } + + public class HDRDatasetUsage + { + public List dataUseLimitation { get; set; } + public object resourceCreator { get; set; } + public List dataUseRequirements { get; set; } = []; + } + public class HDRDatasetAccessibility + { + public HDRDatasetAccess access { get; set; } + public HDRDatasetUsage usage { get; set; } + public HDRDatasetFormatAndStandards formatAndStandards { get; set; } + } + public class HDRDatasetProvenance + { + public HDRDatasetOrigin origin { get; set; } + public HDRDatasetTemporal temporal { get; set; } + } + public class HDRDatasetRevision + { + public string version { get; set; } + public string url { get; set; } + } + + public class HDRDatasetMetadata + { + + public HDRDatasetMetadata(Catalogue catalogue) + { + identifier = ""; + version = "1.0.0"; + modified = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); + issued = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); + summary = new HDRDatasetMetadataSummary(catalogue); + } + public HDRDatasetMetadata() { } + public string identifier = ""; + public string version = "1.0.0"; + public List revisions = new List(); + public string modified; + public string issued; + public HDRDatasetMetadataSummary summary = new HDRDatasetMetadataSummary(); + public HDRDatasetAccessibility accessibility = new HDRDatasetAccessibility(); + public List observations = new List(); + public HDRDatasetProvenance provenance = new HDRDatasetProvenance(); + } + + public class HDRDatasetInterMetadata + { + public HDRDatasetMetadata metadata { get; set; } + public HDRDatasetInterMetadata() { } + public HDRDatasetInterMetadata(Catalogue catalogue) + { + metadata = new HDRDatasetMetadata(catalogue); + } + } + + private class HDRDatasetLatestMetadata + { + public int id { get; set; } + public HDRDatasetInterMetadata metadata { get; set; } + + public HDRDatasetLatestMetadata() { } + public HDRDatasetLatestMetadata(Catalogue catalogue) + { + metadata = new HDRDatasetInterMetadata(catalogue); + } + + } + + private class HDRDatasetObject + { + public int id { get; set; } + public int team_id { get; set; } + public HDRDatasetLatestMetadata latest_metadata { get; set; } + } + + private class HDRResponse + { + public int current_page { get; set; } + public List data { get; set; } + } + + private class HDRDatasetCreationBody + { + public HDRDatasetMetadata metadata = new(); + public HDRDatasetCreationBody(Catalogue catalogue) + { + metadata = new HDRDatasetMetadata(catalogue); + } + } + + public ExecuteCommandExportCataloguesToHDRGateway(IBasicActivateItems activator, string url, int team, string appID, string clientID) + { + _activator = activator; + _url = url.TrimEnd('/'); + _team = team; + _appID = appID; + _clientID = clientID; + } + + private bool CatalogueExists(Catalogue catalogue) + { + var url = $"{_url}{_datasetsEndpoint}?title={catalogue.Name}&team={_team}"; + var response = Task.Run(async () => await _client.GetAsync(url)).Result; + var contentString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + var content = JsonConvert.DeserializeObject(contentString); + if (!content.data.Any()) return false; + if (content.data.Any(dataset => dataset.latest_metadata.metadata.metadata.summary.title == catalogue.Name)) return true; + return false; + } + + private void CreateDataset(Catalogue catalogue) + { + var url = $"{_url}/v2/teams/{_team}/datasets?input_schema=HDRUK&input_version=3.0.0?input_schema=HDRUK&input_version=3.0.0"; + HDRDatasetCreationBody body = new HDRDatasetCreationBody(catalogue); + var serializeOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + IncludeFields = true, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + var jsonString = "{\"metadata\":{\"identifier\":\"05ec5a13-3955-45a3-b449-8aba78622113\",\"issued\":\"2025-01-08T11:23:35.000Z\",\"modified\":\"2025-01-08T11:23:35.000Z\",\"revisions\":[{\"version\":\"1.0.0\",\"url\":\"http://www.example.com/\"}],\"version\":\"1.0.0\",\"summary\":{\"abstract\":\"Thisisatestdatasetabstract-update11\",\"contactPoint\":\"jfriel001@dundee.ac.uk\",\"keywords\":[],\"doiName\":null,\"title\":\"JFRIELTESTDataset\",\"dataCustodian\":{\"name\":\"HealthInformaticsCentre-UniversityofDundee\",\"identifier\":\"12\",\"contactPoint\":\"jfriel001@dundee.ac.uk\",\"logo\":null,\"description\":null,\"memberOf\":null},\"populationSize\":-1,\"alternateIdentifiers\":null},\"documentation\":{\"description\":\"TESTDescription3\",\"associatedMedia\":null,\"inPipeline\":\"Notavailable\"},\"coverage\":{\"pathway\":null,\"spatial\":\"Prussia,OttomanEmpire,GranColombia\",\"followUp\":null,\"datasetCompleteness\":null,\"materialType\":[\"Other\"],\"typicalAgeRangeMin\":0,\"typicalAgeRangeMax\":0},\"provenance\":{\"origin\":{\"purpose\":[],\"source\":[],\"collectionSource\":[],\"datasetType\":[\"Healthanddisease\"],\"datasetSubType\":[\"Notapplicable\"],\"imageContrast\":\"Notstated\"},\"temporal\":{\"endDate\":null,\"startDate\":\"2025-01-01T11:26:16.000Z\",\"timeLag\":\"Variable\",\"publishingFrequency\":\"Irregular\",\"distributionReleaseDate\":null}},\"accessibility\":{\"access\":{\"deliveryLeadTime\":\"Notapplicable\",\"jurisdiction\":[\"UK\",\"SC\"],\"dataController\":\"HealthInformaticsCentre-UniversityofDundee\",\"dataProcessor\":\"HealthInformaticsCentre-UniversityofDundee\",\"accessRights\":\"https://www.dundee.ac.uk/hic/governance-service\",\"accessService\":\"HIChasimplementedaremote-accessTrustedResearchEnvironmenttoprotectdataconfidentiality,satisfypublicconcernsaboutdatalossandreassureDataControllersaboutHIC\\u00e2\\u20ac\\u2122ssecuremanagementandprocessingoftheirdata.\\n\\nDataisnotreleasedexternallytodatausersforanalysisontheirowncomputersbutplacedonaserveratHIC,withinarestricted,secureITenvironment,wherethedatauserisgivensecureremoteaccesstocarryouttheiranalysis.\\n\\nFulldetailsareavailableviathefollowinglink:\\nhttps://www.dundee.ac.uk/hic/safe-haven\",\"accessRequestCost\":\"Quotationavailableonrequest\",\"accessServiceCategory\":null},\"usage\":{\"dataUseLimitation\":[\"Generalresearchuse\"],\"resourceCreator\":null,\"dataUseRequirements\":null},\"formatAndStandards\":{\"conformsTo\":[\"I2B2\"],\"vocabularyEncodingScheme\":[\"LOCAL\"],\"language\":[\"en\"],\"format\":[\"CSV\",\"Database\"]}},\"enrichmentAndLinkage\":{\"tools\":[],\"investigations\":[],\"publicationAboutDataset\":[],\"publicationUsingDataset\":[],\"derivedFrom\":null,\"isPartOf\":null,\"linkableDatasets\":null,\"similarToDatasets\":null},\"observations\":[],\"structuralMetadata\":{\"tables\":[],\"syntheticDataWebLink\":null},\"demographicFrequency\":null,\"omics\":null}}";//System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + var response = Task.Run(async () => await _client.PostAsync(url, httpContent)).Result; + var contentString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + var content = JsonConvert.DeserializeObject(contentString); + Console.WriteLine($"Created Dataset '{catalogue.Name}'"); + + } + + public override void Execute() + { + base.Execute(); + _client.DefaultRequestHeaders.Add("x-application-id", _appID); + _client.DefaultRequestHeaders.Add("x-client-id", _clientID); + + var catalogues = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects() + .Where(c => !c.IsDeprecated && !c.IsInternalDataset && !c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) + .ToList(); + foreach (var catalogue in catalogues) + { + if (CatalogueExists(catalogue)) + { + //update + Console.WriteLine($"update {catalogue.Name}"); + } + else + { + //create + CreateDataset(catalogue); + } + } + + } + + } +} From 4be2764ee37527a23b4ab223384c7795409f2ed4 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 10 Dec 2025 09:16:58 +0000 Subject: [PATCH 02/13] workign create --- ...cuteCommandExportCataloguesToHDRGateway.cs | 206 ++++++++++++++---- 1 file changed, 165 insertions(+), 41 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs index c425e1c422..88bb876f44 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs @@ -5,8 +5,10 @@ using Newtonsoft.Json; using NPOI.POIFS.Crypt; using NPOI.SS.UserModel.Charts; +using NPOI.XWPF.UserModel; using Rdmp.Core.Curation.Data; using Rdmp.Core.Dataset.Confluence; +using Renci.SshNet.Sftp; using System; using System.Collections.Generic; using System.Globalization; @@ -33,31 +35,67 @@ public class ExecuteCommandExportCataloguesToHDRGateway : BasicCommandExecution, private readonly HttpClient _client = new(); private readonly string _datasetsEndpoint = "/v2/datasets"; + public class HDRDatasetDataCustodian + { + public HDRDatasetDataCustodian() { } + public HDRDatasetDataCustodian(Catalogue catalogue) + { + //todo check this + name = catalogue.DataController; + contactPoint = catalogue.Contact_details; + } + public string name { get; set; }//this needs to be populated + public int identifier { get; set; } = 0; + public string contactPoint { get; set; }//this needs to be populated + public object logo { get; set; } + public object description { get; set; } + public object memberOf { get; set; } + } + + public class HDRDatasetDocumentation + { + public HDRDatasetDocumentation() { } + public HDRDatasetDocumentation(Catalogue catalogue) + { + description = catalogue.Description; + } + public string description { get; set; }//this needs to be populated; + } public class HDRDatasetMetadataSummary { + public HDRDatasetMetadataSummary(Catalogue catalogue) { title = catalogue.Name; populationSize = 0; - contactPoint = ""; + @abstract = catalogue.ShortDescription; + contactPoint = catalogue.Administrative_contact_email; + dataCustodian = new HDRDatasetDataCustodian(catalogue); } public HDRDatasetMetadataSummary() { } - public string @abstract { get; set; } - public string contactPoint { get; set; } + public string @abstract { get; set; }//this needs to be populated + public string contactPoint { get; set; }//this needs to be populated public List keywords { get; set; } public object doiName { get; set; } public string title { get; set; } - //public DataCustodian dataCustodian { get; set; } + public HDRDatasetDataCustodian dataCustodian { get; set; } = new HDRDatasetDataCustodian(); public int populationSize { get; set; } public object alternateIdentifiers { get; set; } } public class HDRDatasetAccess { + public HDRDatasetAccess() { } + public HDRDatasetAccess(Catalogue catalogue) + { + //todo access rights + accessRights = catalogue.Access_options; + //jurisdiction = catalogue.Juristiction.Split(',').ToList(); + } public string deliveryLeadTime { get; set; } - public List jurisdiction { get; set; } + //public List jurisdiction { get; set; } public string dataController { get; set; } public string dataProcessor { get; set; } - public string accessRights { get; set; } + public string accessRights { get; set; }//this needs to be populated public string accessService { get; set; } public string accessRequestCost { get; set; } public object accessServiceCategory { get; set; } @@ -67,7 +105,7 @@ private class CustomDateTimeConverterThreeMilliseconds : JsonConverter { public override DateTime ReadJson(JsonReader reader, Type objectType, DateTime existingValue, bool hasExistingValue, Newtonsoft.Json.JsonSerializer serializer) { - return DateTime.Parse(reader.ReadAsString()); + return existingValue; } @@ -79,23 +117,35 @@ public override void WriteJson(JsonWriter writer, DateTime value, Newtonsoft.Jso public class HDRDatasetTemporal { - [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] + public HDRDatasetTemporal() { } + public HDRDatasetTemporal(Catalogue catalogue) + { + timeLag = catalogue.UpdateLag.ToString(); + publishingFrequency = catalogue.Update_freq.ToString(); + } + [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] public DateTime? endDate { get; set; } [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] - public DateTime startDate { get; set; } - public string timeLag { get; set; } - public string publishingFrequency { get; set; } + public DateTime startDate { get; set; } = DateTime.Now; + public string timeLag { get; set; }//this needs to be populated + public string publishingFrequency { get; set; }//this needs to be populated public object distributionReleaseDate { get; set; } } public class HDRDatasetFormatAndStandards { - public List conformsTo { get; set; } - public List vocabularyEncodingScheme { get; set; } - public List language { get; set; } - public List format { get; set; } + public HDRDatasetFormatAndStandards() { } + public HDRDatasetFormatAndStandards(Catalogue catalogue) + { + //todo + } + + public string conformsTo { get; set; } = "LOCAL";// new List() { "LOCAL" }; + public string vocabularyEncodingScheme { get; set; } = "LOCAL";// new List() { "LOCAL" }; + public string language { get; set; } = "en";// new List() { "en" }; + public string format { get; set; } = "CSV;,Database;";//new List() { "CSV", "Database" }; } public class HDRDatasetOrigin @@ -116,14 +166,25 @@ public class HDRDatasetUsage } public class HDRDatasetAccessibility { - public HDRDatasetAccess access { get; set; } + public HDRDatasetAccessibility() { } + public HDRDatasetAccessibility(Catalogue catalogue) + { + //access = new HDRDatasetAccess(catalogue); + formatAndStandards = new HDRDatasetFormatAndStandards(catalogue); + } + //public HDRDatasetAccess access = new HDRDatasetAccess(); public HDRDatasetUsage usage { get; set; } - public HDRDatasetFormatAndStandards formatAndStandards { get; set; } + public HDRDatasetFormatAndStandards formatAndStandards { get; set; } = new HDRDatasetFormatAndStandards(); } public class HDRDatasetProvenance { + public HDRDatasetProvenance() { } + public HDRDatasetProvenance(Catalogue catalogue) + { + temporal = new HDRDatasetTemporal(catalogue); + } public HDRDatasetOrigin origin { get; set; } - public HDRDatasetTemporal temporal { get; set; } + public HDRDatasetTemporal temporal { get; set; } = new HDRDatasetTemporal(); } public class HDRDatasetRevision { @@ -131,12 +192,32 @@ public class HDRDatasetRevision public string url { get; set; } } + public class HDRDatasetCoverage + { + + public HDRDatasetCoverage() { } + public HDRDatasetCoverage(Catalogue catalogue) + { + spatial = catalogue.Country_of_origin; + } + public string spatial { get; set; } = "Prussia"; + } + + public class HDRDatasetRequired{ + public string issued { get; set; } + public string version { get; set; } + public string modified { get; set; } + public string gatewayId { get; set; } + public string gatewayPid { get; set; } + + public List revisions { get; set; } = new(); + } + public class HDRDatasetMetadata { - + public HDRDatasetMetadata(Catalogue catalogue) { - identifier = ""; version = "1.0.0"; modified = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); issued = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); @@ -151,30 +232,21 @@ public HDRDatasetMetadata() { } public HDRDatasetMetadataSummary summary = new HDRDatasetMetadataSummary(); public HDRDatasetAccessibility accessibility = new HDRDatasetAccessibility(); public List observations = new List(); + public HDRDatasetDocumentation documentation = new HDRDatasetDocumentation(); + public HDRDatasetCoverage coverage = new HDRDatasetCoverage(); public HDRDatasetProvenance provenance = new HDRDatasetProvenance(); + public List tissueSampleCollection = new List { }; } public class HDRDatasetInterMetadata { public HDRDatasetMetadata metadata { get; set; } - public HDRDatasetInterMetadata() { } - public HDRDatasetInterMetadata(Catalogue catalogue) - { - metadata = new HDRDatasetMetadata(catalogue); - } } private class HDRDatasetLatestMetadata { public int id { get; set; } public HDRDatasetInterMetadata metadata { get; set; } - - public HDRDatasetLatestMetadata() { } - public HDRDatasetLatestMetadata(Catalogue catalogue) - { - metadata = new HDRDatasetInterMetadata(catalogue); - } - } private class HDRDatasetObject @@ -192,10 +264,12 @@ private class HDRResponse private class HDRDatasetCreationBody { + public string status = "ACTIVE"; + public string create_origin = "MANUAL"; public HDRDatasetMetadata metadata = new(); public HDRDatasetCreationBody(Catalogue catalogue) { - metadata = new HDRDatasetMetadata(catalogue); + metadata = new HDRDatasetMetadata(catalogue); } } @@ -208,15 +282,17 @@ public ExecuteCommandExportCataloguesToHDRGateway(IBasicActivateItems activator, _clientID = clientID; } - private bool CatalogueExists(Catalogue catalogue) + private HDRDatasetObject CatalogueExists(Catalogue catalogue) { - var url = $"{_url}{_datasetsEndpoint}?title={catalogue.Name}&team={_team}"; + var url = $"{_url}{_datasetsEndpoint}?title={catalogue.Name}&team={_team}&with_metadata=true"; var response = Task.Run(async () => await _client.GetAsync(url)).Result; var contentString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; var content = JsonConvert.DeserializeObject(contentString); - if (!content.data.Any()) return false; - if (content.data.Any(dataset => dataset.latest_metadata.metadata.metadata.summary.title == catalogue.Name)) return true; - return false; + if (!content.data.Any()) return null; + var found = content.data.FirstOrDefault(dataset => dataset.latest_metadata.metadata.metadata.summary.title == catalogue.Name); + if (found != null) return found; + return null; + } private void CreateDataset(Catalogue catalogue) @@ -230,13 +306,60 @@ private void CreateDataset(Catalogue catalogue) IncludeFields = true, DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; - var jsonString = "{\"metadata\":{\"identifier\":\"05ec5a13-3955-45a3-b449-8aba78622113\",\"issued\":\"2025-01-08T11:23:35.000Z\",\"modified\":\"2025-01-08T11:23:35.000Z\",\"revisions\":[{\"version\":\"1.0.0\",\"url\":\"http://www.example.com/\"}],\"version\":\"1.0.0\",\"summary\":{\"abstract\":\"Thisisatestdatasetabstract-update11\",\"contactPoint\":\"jfriel001@dundee.ac.uk\",\"keywords\":[],\"doiName\":null,\"title\":\"JFRIELTESTDataset\",\"dataCustodian\":{\"name\":\"HealthInformaticsCentre-UniversityofDundee\",\"identifier\":\"12\",\"contactPoint\":\"jfriel001@dundee.ac.uk\",\"logo\":null,\"description\":null,\"memberOf\":null},\"populationSize\":-1,\"alternateIdentifiers\":null},\"documentation\":{\"description\":\"TESTDescription3\",\"associatedMedia\":null,\"inPipeline\":\"Notavailable\"},\"coverage\":{\"pathway\":null,\"spatial\":\"Prussia,OttomanEmpire,GranColombia\",\"followUp\":null,\"datasetCompleteness\":null,\"materialType\":[\"Other\"],\"typicalAgeRangeMin\":0,\"typicalAgeRangeMax\":0},\"provenance\":{\"origin\":{\"purpose\":[],\"source\":[],\"collectionSource\":[],\"datasetType\":[\"Healthanddisease\"],\"datasetSubType\":[\"Notapplicable\"],\"imageContrast\":\"Notstated\"},\"temporal\":{\"endDate\":null,\"startDate\":\"2025-01-01T11:26:16.000Z\",\"timeLag\":\"Variable\",\"publishingFrequency\":\"Irregular\",\"distributionReleaseDate\":null}},\"accessibility\":{\"access\":{\"deliveryLeadTime\":\"Notapplicable\",\"jurisdiction\":[\"UK\",\"SC\"],\"dataController\":\"HealthInformaticsCentre-UniversityofDundee\",\"dataProcessor\":\"HealthInformaticsCentre-UniversityofDundee\",\"accessRights\":\"https://www.dundee.ac.uk/hic/governance-service\",\"accessService\":\"HIChasimplementedaremote-accessTrustedResearchEnvironmenttoprotectdataconfidentiality,satisfypublicconcernsaboutdatalossandreassureDataControllersaboutHIC\\u00e2\\u20ac\\u2122ssecuremanagementandprocessingoftheirdata.\\n\\nDataisnotreleasedexternallytodatausersforanalysisontheirowncomputersbutplacedonaserveratHIC,withinarestricted,secureITenvironment,wherethedatauserisgivensecureremoteaccesstocarryouttheiranalysis.\\n\\nFulldetailsareavailableviathefollowinglink:\\nhttps://www.dundee.ac.uk/hic/safe-haven\",\"accessRequestCost\":\"Quotationavailableonrequest\",\"accessServiceCategory\":null},\"usage\":{\"dataUseLimitation\":[\"Generalresearchuse\"],\"resourceCreator\":null,\"dataUseRequirements\":null},\"formatAndStandards\":{\"conformsTo\":[\"I2B2\"],\"vocabularyEncodingScheme\":[\"LOCAL\"],\"language\":[\"en\"],\"format\":[\"CSV\",\"Database\"]}},\"enrichmentAndLinkage\":{\"tools\":[],\"investigations\":[],\"publicationAboutDataset\":[],\"publicationUsingDataset\":[],\"derivedFrom\":null,\"isPartOf\":null,\"linkableDatasets\":null,\"similarToDatasets\":null},\"observations\":[],\"structuralMetadata\":{\"tables\":[],\"syntheticDataWebLink\":null},\"demographicFrequency\":null,\"omics\":null}}";//System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); + var jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PostAsync(url, httpContent)).Result; + if (response.StatusCode != System.Net.HttpStatusCode.Created) + { + //couldn't create, so will try draft - maybe the metadata is missing something + body.status = "DRAFT"; + jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); + httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + response = Task.Run(async () => await _client.PostAsync(url, httpContent)).Result; + if(response.StatusCode != System.Net.HttpStatusCode.Created) + { + Console.WriteLine($"Unable to create dataset for {catalogue.Name}"); + return; + } + } var contentString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; var content = JsonConvert.DeserializeObject(contentString); Console.WriteLine($"Created Dataset '{catalogue.Name}'"); + } + + private void UpdateDataset(Catalogue catalogue,HDRDatasetObject existingObject) + { + var url = $"{_url}/v2/teams/{_team}/datasets/{existingObject.id}?input_schema=HDRUK&input_version=3.0.0?input_schema=HDRUK&input_version=3.0.0"; + var body = existingObject.latest_metadata; + //HDRDatasetCreationBody body = new HDRDatasetCreationBody(catalogue); + var serializeOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + IncludeFields = true, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + //body.metadata.summary.title = body.metadata.summary.title + new Random().NextInt64(30); + var jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + var response = Task.Run(async () => await _client.PatchAsync(url, httpContent)).Result; + //if (response.StatusCode != System.Net.HttpStatusCode.Created) + //{ + // //couldn't create, so will try draft - maybe the metadata is missing something + // body.status = "DRAFT"; + // jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); + // httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + // response = Task.Run(async () => await _client.PostAsync(url, httpContent)).Result; + // if (response.StatusCode != System.Net.HttpStatusCode.Created) + // { + // Console.WriteLine($"Unable to create dataset for {catalogue.Name}"); + // return; + // } + //} + var contentString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + var content = JsonConvert.DeserializeObject(contentString); + Console.WriteLine($"Updated Dataset '{catalogue.Name}'"); } public override void Execute() @@ -250,10 +373,11 @@ public override void Execute() .ToList(); foreach (var catalogue in catalogues) { - if (CatalogueExists(catalogue)) + var existingDataset = CatalogueExists(catalogue); + if (existingDataset != null) { //update - Console.WriteLine($"update {catalogue.Name}"); + UpdateDataset(catalogue,existingDataset); } else { From 473929d137b647445f2b274071057f576a79ed22 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 12 Dec 2025 09:31:55 +0000 Subject: [PATCH 03/13] update working --- ...cuteCommandExportCataloguesToHDRGateway.cs | 193 +++++++++++++++--- 1 file changed, 159 insertions(+), 34 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs index 88bb876f44..1296fbda14 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs @@ -61,9 +61,12 @@ public HDRDatasetDocumentation(Catalogue catalogue) } public string description { get; set; }//this needs to be populated; } - public class HDRDatasetMetadataSummary + private class HDRDatasetMetadataSummary { - + public HDRDatasetMetadataSummary(HDRDatasetObject datasetObject) + { + //todo + } public HDRDatasetMetadataSummary(Catalogue catalogue) { title = catalogue.Name; @@ -81,6 +84,7 @@ public HDRDatasetMetadataSummary() { } public HDRDatasetDataCustodian dataCustodian { get; set; } = new HDRDatasetDataCustodian(); public int populationSize { get; set; } public object alternateIdentifiers { get; set; } + public string controlledKeywords { get; set; } } public class HDRDatasetAccess { @@ -132,6 +136,8 @@ public HDRDatasetTemporal(Catalogue catalogue) public string timeLag { get; set; }//this needs to be populated public string publishingFrequency { get; set; }//this needs to be populated public object distributionReleaseDate { get; set; } + + public string accrualPeriodicity { get; set; }//this is the same as publishign frequency? } public class HDRDatasetFormatAndStandards @@ -142,10 +148,10 @@ public HDRDatasetFormatAndStandards(Catalogue catalogue) //todo } - public string conformsTo { get; set; } = "LOCAL";// new List() { "LOCAL" }; - public string vocabularyEncodingScheme { get; set; } = "LOCAL";// new List() { "LOCAL" }; - public string language { get; set; } = "en";// new List() { "en" }; - public string format { get; set; } = "CSV;,Database;";//new List() { "CSV", "Database" }; + public string conformsTo { get; set; } = "LOCAL"; + public string vocabularyEncodingSchemes { get; set; } = "LOCAL"; + public string languages { get; set; } = "en"; + public string formats { get; set; } = "CSV;,Database;"; } public class HDRDatasetOrigin @@ -164,21 +170,29 @@ public class HDRDatasetUsage public object resourceCreator { get; set; } public List dataUseRequirements { get; set; } = []; } - public class HDRDatasetAccessibility + private class HDRDatasetAccessibility { public HDRDatasetAccessibility() { } + public HDRDatasetAccessibility(HDRDatasetObject datasetObject) + { + //todo + } public HDRDatasetAccessibility(Catalogue catalogue) { //access = new HDRDatasetAccess(catalogue); formatAndStandards = new HDRDatasetFormatAndStandards(catalogue); } - //public HDRDatasetAccess access = new HDRDatasetAccess(); + public object access { get; set; }// = new HDRDatasetAccess();//todo there is an issue descerialising this public HDRDatasetUsage usage { get; set; } public HDRDatasetFormatAndStandards formatAndStandards { get; set; } = new HDRDatasetFormatAndStandards(); } - public class HDRDatasetProvenance + private class HDRDatasetProvenance { public HDRDatasetProvenance() { } + public HDRDatasetProvenance(HDRDatasetObject datasetObject) + { + //todo + } public HDRDatasetProvenance(Catalogue catalogue) { temporal = new HDRDatasetTemporal(catalogue); @@ -192,10 +206,14 @@ public class HDRDatasetRevision public string url { get; set; } } - public class HDRDatasetCoverage + private class HDRDatasetCoverage { public HDRDatasetCoverage() { } + public HDRDatasetCoverage(HDRDatasetObject datasetObject) + { + //todo + } public HDRDatasetCoverage(Catalogue catalogue) { spatial = catalogue.Country_of_origin; @@ -203,7 +221,14 @@ public HDRDatasetCoverage(Catalogue catalogue) public string spatial { get; set; } = "Prussia"; } - public class HDRDatasetRequired{ + private class HDRDatasetRequired + { + + public HDRDatasetRequired() { } + public HDRDatasetRequired(HDRDatasetObject datasetObject) + { + //todo + } public string issued { get; set; } public string version { get; set; } public string modified { get; set; } @@ -213,7 +238,7 @@ public class HDRDatasetRequired{ public List revisions { get; set; } = new(); } - public class HDRDatasetMetadata + private class HDRDatasetMetadata { public HDRDatasetMetadata(Catalogue catalogue) @@ -236,9 +261,10 @@ public HDRDatasetMetadata() { } public HDRDatasetCoverage coverage = new HDRDatasetCoverage(); public HDRDatasetProvenance provenance = new HDRDatasetProvenance(); public List tissueSampleCollection = new List { }; + public HDRDatasetRequired required { get; set; } } - public class HDRDatasetInterMetadata + private class HDRDatasetInterMetadata { public HDRDatasetMetadata metadata { get; set; } } @@ -273,6 +299,108 @@ public HDRDatasetCreationBody(Catalogue catalogue) } } + private class HDRDatasetUpdateMetadata + { + public HDRDatasetMetadataSummary summary { get; set; } + public HDRDatasetCoverage coverage { get; set; } + public HDRDatasetRequired required { get; set; } + public HDRDatasetProvenance provenance { get; set; } + public List observations = []; + public HDRDatasetAccessibility accessibility { get; set; } + public List tissuesSampleCollection = []; + + public HDRDatasetUpdateMetadata() { } + public HDRDatasetUpdateMetadata(HDRDatasetObject datasetObject) + { + summary = datasetObject.latest_metadata.metadata.metadata.summary;//new HDRDatasetMetadataSummary(datasetObject); + coverage = datasetObject.latest_metadata.metadata.metadata.coverage;// new HDRDatasetCoverage(datasetObject); + required = datasetObject.latest_metadata.metadata.metadata.required; //new HDRDatasetRequired(datasetObject); + provenance = datasetObject.latest_metadata.metadata.metadata.provenance; //new HDRDatasetProvenance(datasetObject); + accessibility = datasetObject.latest_metadata.metadata.metadata.accessibility; //new HDRDatasetAccessibility(datasetObject); + } + } + + private class HDRDatasetUpdateInterMetadata + { + + public HDRDatasetUpdateInterMetadata(HDRDatasetObject datasetObject) + { + metadata = new HDRDatasetUpdateMetadata(datasetObject); + } + + public HDRDatasetUpdateMetadata metadata { get; set; } + } + + private class HDRDatasetUpdateBody + { + public string status = "ACTIVE"; + public HDRDatasetUpdateInterMetadata metadata { get; set; } + + public HDRDatasetUpdateBody() { } + public HDRDatasetUpdateBody(HDRDatasetObject datasetObject) + { + metadata = new HDRDatasetUpdateInterMetadata(datasetObject); + } + + } + + private class HDRDatasetPUTBody + { + public string status = "ACTIVE"; + public string create_origin = "MANUAL"; + //todo put metadata/metadata in here for put specific + + // { + // "status":"ACTIVE", + // "metadata": { + // "metadata": { + // "summary": { + // "title": "jf_Biochemistry3", + // "abstract": "12345", + // "controlledKeywords": null + // }, + // "coverage": { + // "spatial": "Prussia" + // }, + // "required": { + // "issued": "2025-12-09T14:53:07.260524Z", + // "version": "1.0.0", + // "modified": "2025-12-09T14:53:07.260541Z", + // "gatewayId": "1995", + // "revisions": [ + // { + // "url": "https://web.preprod.hdruk.cloud/dataset/1995?version=1.0.0", + // "version": "1.0.0" + // } + // ], + // "gatewayPid": "7e01763e-3d8f-4801-8fe1-548ede6e2f4e" + // }, + // "provenance": { + // "temporal": { + // "timeLag": "Less than 1 week", + // "startDate": "2025-12-09T14:53:06.9626377+00:00", + // "accrualPeriodicity": "Quarterly", + // "distributionReleaseDate": null + // } + // }, + // "observations": [], + // "accessibility": { + // "access": { + // "accessRights": "123" + // }, + // "formatAndStandards": { + // "formats": "CSV", + // "languages": "en", + // "conformsTo": "HL7 FHIR", + // "vocabularyEncodingSchemes": "LOCAL" + // } + //}, + // "tissuesSampleCollection": [] + // } + // } + //} + } + public ExecuteCommandExportCataloguesToHDRGateway(IBasicActivateItems activator, string url, int team, string appID, string clientID) { _activator = activator; @@ -316,7 +444,7 @@ private void CreateDataset(Catalogue catalogue) jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); response = Task.Run(async () => await _client.PostAsync(url, httpContent)).Result; - if(response.StatusCode != System.Net.HttpStatusCode.Created) + if (response.StatusCode != System.Net.HttpStatusCode.Created) { Console.WriteLine($"Unable to create dataset for {catalogue.Name}"); return; @@ -327,12 +455,11 @@ private void CreateDataset(Catalogue catalogue) Console.WriteLine($"Created Dataset '{catalogue.Name}'"); } - private void UpdateDataset(Catalogue catalogue,HDRDatasetObject existingObject) + private void UpdateDataset(Catalogue catalogue, HDRDatasetObject existingObject) { var url = $"{_url}/v2/teams/{_team}/datasets/{existingObject.id}?input_schema=HDRUK&input_version=3.0.0?input_schema=HDRUK&input_version=3.0.0"; - var body = existingObject.latest_metadata; - //HDRDatasetCreationBody body = new HDRDatasetCreationBody(catalogue); + HDRDatasetUpdateBody body = new HDRDatasetUpdateBody(existingObject); var serializeOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -340,25 +467,23 @@ private void UpdateDataset(Catalogue catalogue,HDRDatasetObject existingObject) IncludeFields = true, DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; - //body.metadata.summary.title = body.metadata.summary.title + new Random().NextInt64(30); + body.metadata.metadata.summary.dataCustodian = null; + body.metadata.metadata.summary.@abstract = "todo123"; + body.metadata.metadata.summary.controlledKeywords = "this;,that;"; + body.metadata.metadata.provenance.temporal.timeLag = "Less than 1 week"; + body.metadata.metadata.provenance.temporal.distributionReleaseDate = null; + body.metadata.metadata.provenance.temporal.accrualPeriodicity = "Quarterly"; + body.metadata.metadata.accessibility.access = new HDRDatasetAccess() { accessRights="123"}; + body.metadata.metadata.accessibility.usage = null; var jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PatchAsync(url, httpContent)).Result; - //if (response.StatusCode != System.Net.HttpStatusCode.Created) - //{ - // //couldn't create, so will try draft - maybe the metadata is missing something - // body.status = "DRAFT"; - // jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); - // httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); - // response = Task.Run(async () => await _client.PostAsync(url, httpContent)).Result; - // if (response.StatusCode != System.Net.HttpStatusCode.Created) - // { - // Console.WriteLine($"Unable to create dataset for {catalogue.Name}"); - // return; - // } - //} - var contentString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - var content = JsonConvert.DeserializeObject(contentString); + if (response.StatusCode != System.Net.HttpStatusCode.OK) + { + var contentString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + Console.WriteLine($"unable to update Dataset '{catalogue.Name}': {contentString}"); + + } Console.WriteLine($"Updated Dataset '{catalogue.Name}'"); } @@ -377,7 +502,7 @@ public override void Execute() if (existingDataset != null) { //update - UpdateDataset(catalogue,existingDataset); + UpdateDataset(catalogue, existingDataset); } else { From eebab1b022247aa0084bd3a3dfd5e416f6005077 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 15 Dec 2025 14:22:34 +0000 Subject: [PATCH 04/13] working create --- ...cuteCommandExportCataloguesToHDRGateway.cs | 247 ++++++++++-------- 1 file changed, 139 insertions(+), 108 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs index 1296fbda14..2ff5b7ef96 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs @@ -17,6 +17,7 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading.Tasks; namespace Rdmp.Core.CommandExecution.AtomicCommands @@ -40,9 +41,8 @@ public class HDRDatasetDataCustodian public HDRDatasetDataCustodian() { } public HDRDatasetDataCustodian(Catalogue catalogue) { - //todo check this - name = catalogue.DataController; - contactPoint = catalogue.Contact_details; + name = catalogue.DataProcessor; + contactPoint = catalogue.Administrative_contact_email; } public string name { get; set; }//this needs to be populated public int identifier { get; set; } = 0; @@ -63,10 +63,6 @@ public HDRDatasetDocumentation(Catalogue catalogue) } private class HDRDatasetMetadataSummary { - public HDRDatasetMetadataSummary(HDRDatasetObject datasetObject) - { - //todo - } public HDRDatasetMetadataSummary(Catalogue catalogue) { title = catalogue.Name; @@ -74,6 +70,8 @@ public HDRDatasetMetadataSummary(Catalogue catalogue) @abstract = catalogue.ShortDescription; contactPoint = catalogue.Administrative_contact_email; dataCustodian = new HDRDatasetDataCustodian(catalogue); + controlledKeywords = catalogue.Search_keywords; + doiName = catalogue.Doi; } public HDRDatasetMetadataSummary() { } public string @abstract { get; set; }//this needs to be populated @@ -88,15 +86,43 @@ public HDRDatasetMetadataSummary() { } } public class HDRDatasetAccess { + + private string mapUpdateLag(Catalogue.UpdateLagTimes time) + { + switch (time) + { + case Catalogue.UpdateLagTimes.LessThanAWeek: + return "Less than 1 week"; + case Catalogue.UpdateLagTimes.OneToTwoWeeks: + return "1-2 weeks"; + case Catalogue.UpdateLagTimes.TwoToFourWeeks: + return "2-4 weeks"; + case Catalogue.UpdateLagTimes.OneToTwoMonths: + return "1-2 months"; + case Catalogue.UpdateLagTimes.TwoToSixMonths: + return "2-6 months"; + case Catalogue.UpdateLagTimes.SixMonthsPlus: + return "More than 6 months"; + case Catalogue.UpdateLagTimes.Variable: + return "Variable"; + case Catalogue.UpdateLagTimes.NotApplicable: + return "Not applicable"; + case Catalogue.UpdateLagTimes.Other: + return "Other"; + } + return null; + } + public HDRDatasetAccess() { } public HDRDatasetAccess(Catalogue catalogue) { //todo access rights - accessRights = catalogue.Access_options; - //jurisdiction = catalogue.Juristiction.Split(',').ToList(); + accessRights = "TODO";// catalogue.Access_options; + jurisdiction = catalogue.Juristiction.Split(',').ToList(); + deliveryLeadTime = mapUpdateLag(catalogue.UpdateLag); } public string deliveryLeadTime { get; set; } - //public List jurisdiction { get; set; } + public object jurisdiction { get; set; } public string dataController { get; set; } public string dataProcessor { get; set; } public string accessRights { get; set; }//this needs to be populated @@ -148,19 +174,89 @@ public HDRDatasetFormatAndStandards(Catalogue catalogue) //todo } - public string conformsTo { get; set; } = "LOCAL"; - public string vocabularyEncodingSchemes { get; set; } = "LOCAL"; - public string languages { get; set; } = "en"; - public string formats { get; set; } = "CSV;,Database;"; + public object conformsTo { get; set; } = new List() { "LOCAL" }; + public object vocabularyEncodingScheme { get; set; } = new List() { "LOCAL" }; + public object language { get; set; } = new List() { "en" }; + public object format { get; set; } = new List() { "CSV", "Database" }; } public class HDRDatasetOrigin { - public List purpose { get; set; } - public List source { get; set; } - public List collectionSource { get; set; } - public List datasetType { get; set; } - public List datasetSubType { get; set; } + + + public HDRDatasetOrigin() { } + + + private string mapDatasetType(string datasetType) + { + switch (datasetType) + { + case "HealthcareAndDisease": + return "Health and disease"; + case "TreatmentsAndInterventions": + return "Treatments/Interventions"; + case "MeasurementsAndTests": + return "Measurements/Tests"; + case "ImagingTypes": + return "Imaging Types"; + case "Omics": + return "Omics"; + case "Socioeconomic": + return "Socioeconomic"; + case "Lifestyle": + return "Lifestyle"; + case "Registry": + return "Registry"; + case "EnvironmentalAndEnergy": + return "Environment and energy"; + case "InformationAndCommunication": + return "Information and communication"; + case "Politics": + return "Politics"; + } + return ""; + } + + private string mapDataSourceType(string dataSourceType) + { + switch (dataSourceType) + { + case "Other": + return "Other"; + case "EPR": + return "EPR"; + case "ElectronicSurvey": + return "Electronic survey"; + case "LIMS": + return "LIMS"; + case "PaperBased": + return "Paper-based"; + case "FreeTextNLP": + return "TODO"; + case "MachineLearning": + return "Machine generated"; + } + return ""; + } + private class DataTypeObject + { + public string name; + public List subTypes = new(); + } + + public HDRDatasetOrigin(Catalogue catalogue) + { + source = catalogue.DataSource.Split(',').Select(s => mapDataSourceType(s)).Where(s => s != ""); + //collectionSource = catalogue.Source_of_data_collection != null? string.Join(";,", catalogue.Source_of_data_collection.Split(',')) + ';':null; + datasetType = catalogue.DataType != null ? catalogue.DataType.Split(',').Select(s => mapDatasetType(s)).Where(s => s != "").Select(s => new DataTypeObject() { name = s }) : null; //catalogue.DataType;//this has been updated + Console.WriteLine('w'); + //datasetSubType = catalogue.DataSubType != null? string.Join(";,", catalogue.DataSubType.Split(',')) + ';':null; //catalogue.DataSubType; + } + public string purpose { get; set; } + public object source { get; set; } + public string collectionSource { get; set; } + public object datasetType { get; set; }//sometimes string ,somelimes list + public string datasetSubType { get; set; } public string imageContrast { get; set; } } @@ -173,29 +269,24 @@ public class HDRDatasetUsage private class HDRDatasetAccessibility { public HDRDatasetAccessibility() { } - public HDRDatasetAccessibility(HDRDatasetObject datasetObject) - { - //todo - } + public HDRDatasetAccessibility(Catalogue catalogue) { - //access = new HDRDatasetAccess(catalogue); + access = new HDRDatasetAccess(catalogue); formatAndStandards = new HDRDatasetFormatAndStandards(catalogue); } - public object access { get; set; }// = new HDRDatasetAccess();//todo there is an issue descerialising this + public object access { get; set; }// there is an issue descerialising this as it's sometimes an object, sometimes a list public HDRDatasetUsage usage { get; set; } public HDRDatasetFormatAndStandards formatAndStandards { get; set; } = new HDRDatasetFormatAndStandards(); } private class HDRDatasetProvenance { public HDRDatasetProvenance() { } - public HDRDatasetProvenance(HDRDatasetObject datasetObject) - { - //todo - } + public HDRDatasetProvenance(Catalogue catalogue) { temporal = new HDRDatasetTemporal(catalogue); + origin = new HDRDatasetOrigin(catalogue); } public HDRDatasetOrigin origin { get; set; } public HDRDatasetTemporal temporal { get; set; } = new HDRDatasetTemporal(); @@ -210,25 +301,20 @@ private class HDRDatasetCoverage { public HDRDatasetCoverage() { } - public HDRDatasetCoverage(HDRDatasetObject datasetObject) - { - //todo - } + public HDRDatasetCoverage(Catalogue catalogue) { - spatial = catalogue.Country_of_origin; + spatial = catalogue.Geographical_coverage; } public string spatial { get; set; } = "Prussia"; + //TODO - min age, max age, coverage, pathway } private class HDRDatasetRequired { public HDRDatasetRequired() { } - public HDRDatasetRequired(HDRDatasetObject datasetObject) - { - //todo - } + public string issued { get; set; } public string version { get; set; } public string modified { get; set; } @@ -247,6 +333,10 @@ public HDRDatasetMetadata(Catalogue catalogue) modified = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); issued = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); summary = new HDRDatasetMetadataSummary(catalogue); + documentation = new HDRDatasetDocumentation(catalogue); + provenance = new HDRDatasetProvenance(catalogue); + coverage = new HDRDatasetCoverage(catalogue); + accessibility = new HDRDatasetAccessibility(catalogue); } public HDRDatasetMetadata() { } public string identifier = ""; @@ -260,7 +350,7 @@ public HDRDatasetMetadata() { } public HDRDatasetDocumentation documentation = new HDRDatasetDocumentation(); public HDRDatasetCoverage coverage = new HDRDatasetCoverage(); public HDRDatasetProvenance provenance = new HDRDatasetProvenance(); - public List tissueSampleCollection = new List { }; + //public List tissueSampleCollection = new List { }; public HDRDatasetRequired required { get; set; } } @@ -312,11 +402,11 @@ private class HDRDatasetUpdateMetadata public HDRDatasetUpdateMetadata() { } public HDRDatasetUpdateMetadata(HDRDatasetObject datasetObject) { - summary = datasetObject.latest_metadata.metadata.metadata.summary;//new HDRDatasetMetadataSummary(datasetObject); - coverage = datasetObject.latest_metadata.metadata.metadata.coverage;// new HDRDatasetCoverage(datasetObject); - required = datasetObject.latest_metadata.metadata.metadata.required; //new HDRDatasetRequired(datasetObject); - provenance = datasetObject.latest_metadata.metadata.metadata.provenance; //new HDRDatasetProvenance(datasetObject); - accessibility = datasetObject.latest_metadata.metadata.metadata.accessibility; //new HDRDatasetAccessibility(datasetObject); + summary = datasetObject.latest_metadata.metadata.metadata.summary; + coverage = datasetObject.latest_metadata.metadata.metadata.coverage; + required = datasetObject.latest_metadata.metadata.metadata.required; + provenance = datasetObject.latest_metadata.metadata.metadata.provenance; + accessibility = datasetObject.latest_metadata.metadata.metadata.accessibility; } } @@ -333,7 +423,7 @@ public HDRDatasetUpdateInterMetadata(HDRDatasetObject datasetObject) private class HDRDatasetUpdateBody { - public string status = "ACTIVE"; + public string status = "DRAFT"; public HDRDatasetUpdateInterMetadata metadata { get; set; } public HDRDatasetUpdateBody() { } @@ -344,63 +434,6 @@ public HDRDatasetUpdateBody(HDRDatasetObject datasetObject) } - private class HDRDatasetPUTBody - { - public string status = "ACTIVE"; - public string create_origin = "MANUAL"; - //todo put metadata/metadata in here for put specific - - // { - // "status":"ACTIVE", - // "metadata": { - // "metadata": { - // "summary": { - // "title": "jf_Biochemistry3", - // "abstract": "12345", - // "controlledKeywords": null - // }, - // "coverage": { - // "spatial": "Prussia" - // }, - // "required": { - // "issued": "2025-12-09T14:53:07.260524Z", - // "version": "1.0.0", - // "modified": "2025-12-09T14:53:07.260541Z", - // "gatewayId": "1995", - // "revisions": [ - // { - // "url": "https://web.preprod.hdruk.cloud/dataset/1995?version=1.0.0", - // "version": "1.0.0" - // } - // ], - // "gatewayPid": "7e01763e-3d8f-4801-8fe1-548ede6e2f4e" - // }, - // "provenance": { - // "temporal": { - // "timeLag": "Less than 1 week", - // "startDate": "2025-12-09T14:53:06.9626377+00:00", - // "accrualPeriodicity": "Quarterly", - // "distributionReleaseDate": null - // } - // }, - // "observations": [], - // "accessibility": { - // "access": { - // "accessRights": "123" - // }, - // "formatAndStandards": { - // "formats": "CSV", - // "languages": "en", - // "conformsTo": "HL7 FHIR", - // "vocabularyEncodingSchemes": "LOCAL" - // } - //}, - // "tissuesSampleCollection": [] - // } - // } - //} - } - public ExecuteCommandExportCataloguesToHDRGateway(IBasicActivateItems activator, string url, int team, string appID, string clientID) { _activator = activator; @@ -425,7 +458,7 @@ private HDRDatasetObject CatalogueExists(Catalogue catalogue) private void CreateDataset(Catalogue catalogue) { - var url = $"{_url}/v2/teams/{_team}/datasets?input_schema=HDRUK&input_version=3.0.0?input_schema=HDRUK&input_version=3.0.0"; + var url = $"{_url}/v2/teams/{_team}/datasets?input_schema=HDRUK&input_version=4.0.0?input_schema=HDRUK&input_version=4.0.0"; HDRDatasetCreationBody body = new HDRDatasetCreationBody(catalogue); var serializeOptions = new JsonSerializerOptions { @@ -450,14 +483,12 @@ private void CreateDataset(Catalogue catalogue) return; } } - var contentString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - var content = JsonConvert.DeserializeObject(contentString); Console.WriteLine($"Created Dataset '{catalogue.Name}'"); } private void UpdateDataset(Catalogue catalogue, HDRDatasetObject existingObject) { - var url = $"{_url}/v2/teams/{_team}/datasets/{existingObject.id}?input_schema=HDRUK&input_version=3.0.0?input_schema=HDRUK&input_version=3.0.0"; + var url = $"{_url}/v2/teams/{_team}/datasets/{existingObject.id}?input_schema=HDRUK&input_version=4.0.0?input_schema=HDRUK&input_version=4.0.0"; HDRDatasetUpdateBody body = new HDRDatasetUpdateBody(existingObject); var serializeOptions = new JsonSerializerOptions @@ -472,8 +503,8 @@ private void UpdateDataset(Catalogue catalogue, HDRDatasetObject existingObject) body.metadata.metadata.summary.controlledKeywords = "this;,that;"; body.metadata.metadata.provenance.temporal.timeLag = "Less than 1 week"; body.metadata.metadata.provenance.temporal.distributionReleaseDate = null; - body.metadata.metadata.provenance.temporal.accrualPeriodicity = "Quarterly"; - body.metadata.metadata.accessibility.access = new HDRDatasetAccess() { accessRights="123"}; + body.metadata.metadata.provenance.temporal.accrualPeriodicity = body.metadata.metadata.provenance.temporal.publishingFrequency;//todo check this? + body.metadata.metadata.accessibility.access = new HDRDatasetAccess() { accessRights = "123" }; body.metadata.metadata.accessibility.usage = null; var jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); From 45d8638c004faad2b18a81fc2c79e173450f7353 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 16 Dec 2025 08:28:44 +0000 Subject: [PATCH 05/13] improved create --- ...cuteCommandExportCataloguesToHDRGateway.cs | 222 +++++++++++++++--- 1 file changed, 193 insertions(+), 29 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs index 2ff5b7ef96..8b55317a3e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs @@ -2,12 +2,15 @@ using Amazon.SecurityToken.Model; using Azure; using MathNet.Numerics; +using MongoDB.Driver; using Newtonsoft.Json; +using NPOI.OpenXmlFormats.Dml; using NPOI.POIFS.Crypt; using NPOI.SS.UserModel.Charts; using NPOI.XWPF.UserModel; using Rdmp.Core.Curation.Data; using Rdmp.Core.Dataset.Confluence; +using Rdmp.Core.Repositories; using Renci.SshNet.Sftp; using System; using System.Collections.Generic; @@ -59,32 +62,43 @@ public HDRDatasetDocumentation(Catalogue catalogue) { description = catalogue.Description; } + + public void Update(Catalogue catalogue) + { + description = catalogue.Description; + + } public string description { get; set; }//this needs to be populated; } private class HDRDatasetMetadataSummary { public HDRDatasetMetadataSummary(Catalogue catalogue) + { + Update(catalogue); + } + public void Update(Catalogue catalogue) { title = catalogue.Name; populationSize = 0; @abstract = catalogue.ShortDescription; contactPoint = catalogue.Administrative_contact_email; dataCustodian = new HDRDatasetDataCustodian(catalogue); - controlledKeywords = catalogue.Search_keywords; + keywords = catalogue.Search_keywords != null ? catalogue.Search_keywords.Split(',').ToList() : null; doiName = catalogue.Doi; } + public HDRDatasetMetadataSummary() { } public string @abstract { get; set; }//this needs to be populated public string contactPoint { get; set; }//this needs to be populated - public List keywords { get; set; } + public object keywords { get; set; } public object doiName { get; set; } public string title { get; set; } public HDRDatasetDataCustodian dataCustodian { get; set; } = new HDRDatasetDataCustodian(); public int populationSize { get; set; } public object alternateIdentifiers { get; set; } - public string controlledKeywords { get; set; } + public object controlledKeywords { get; set; } } - public class HDRDatasetAccess + private class HDRDatasetAccess { private string mapUpdateLag(Catalogue.UpdateLagTimes time) @@ -117,10 +131,18 @@ public HDRDatasetAccess() { } public HDRDatasetAccess(Catalogue catalogue) { //todo access rights - accessRights = "TODO";// catalogue.Access_options; + accessRights = " ";// catalogue.Access_options; + jurisdiction = catalogue.Juristiction != null?catalogue.Juristiction.Split(',').ToList():null; + deliveryLeadTime = mapUpdateLag(catalogue.UpdateLag); + dataController = catalogue.DataController; + dataProcessor = catalogue.DataProcessor; + } + + public void Update(Catalogue catalogue) { jurisdiction = catalogue.Juristiction.Split(',').ToList(); deliveryLeadTime = mapUpdateLag(catalogue.UpdateLag); } + public string deliveryLeadTime { get; set; } public object jurisdiction { get; set; } public string dataController { get; set; } @@ -152,13 +174,24 @@ public HDRDatasetTemporal(Catalogue catalogue) { timeLag = catalogue.UpdateLag.ToString(); publishingFrequency = catalogue.Update_freq.ToString(); + if (catalogue.StartDate != null) startDate = (DateTime)catalogue.StartDate; + if (catalogue.EndDate != null) endDate = (DateTime)catalogue.EndDate; + + } + + public void Update(Catalogue catalogue) { + timeLag = catalogue.UpdateLag.ToString(); + publishingFrequency = catalogue.Update_freq.ToString(); + if (catalogue.StartDate != null) startDate = (DateTime)catalogue.StartDate; + if (catalogue.EndDate != null) endDate = (DateTime)catalogue.EndDate; + if(catalogue.DatasetReleaseDate != null) distributionReleaseDate = (DateTime)catalogue.DatasetReleaseDate; } [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] public DateTime? endDate { get; set; } [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] - public DateTime startDate { get; set; } = DateTime.Now; + public DateTime startDate { get; set; }// = DateTime.Now; public string timeLag { get; set; }//this needs to be populated public string publishingFrequency { get; set; }//this needs to be populated public object distributionReleaseDate { get; set; } @@ -173,7 +206,10 @@ public HDRDatasetFormatAndStandards(Catalogue catalogue) { //todo } - + public void Update(Catalogue catalogue) + { + //todo + } public object conformsTo { get; set; } = new List() { "LOCAL" }; public object vocabularyEncodingScheme { get; set; } = new List() { "LOCAL" }; public object language { get; set; } = new List() { "en" }; @@ -238,23 +274,118 @@ private string mapDataSourceType(string dataSourceType) } return ""; } + + private string mapPurpose(Catalogue.DatasetPurpose purpose) + { + switch (purpose) + { + + + case Catalogue.DatasetPurpose.ResearchCohort: + return "Research cohort"; + case Catalogue.DatasetPurpose.Study: + return "Study"; + case Catalogue.DatasetPurpose.DiseaseRegistry: + return "Disease registry"; + case Catalogue.DatasetPurpose.Trial: + return "Trial"; + case Catalogue.DatasetPurpose.Care: + return "Care"; + case Catalogue.DatasetPurpose.Audit: + return "Audit"; + case Catalogue.DatasetPurpose.Administrative: + return "Administrative"; + case Catalogue.DatasetPurpose.Finantial: + return "Financial"; + case Catalogue.DatasetPurpose.Statutory: + return "Statutory"; + case Catalogue.DatasetPurpose.Other: + return "Other"; + } + return null; + } + private class DataTypeObject { public string name; public List subTypes = new(); } + private string mapCollectionSource(string source) + { + switch (source) { + case "CohortStudyTrial": + return "Cohort, study, trial"; + case "Clininc": + return "Clinic"; + case "PrimaryCareReferrals": + return "Primary care - Referrals"; + case "PrimaryCareClinic": + return "Primary care - Clinic"; + case "PrimaryCareOutOfHours": + return "Primary care - Out of hours"; + case "SecondaryCareAccidentAndEmergency": + return "Secondary care - Accident and Emergency"; + case "SecondaryCareOutpatients": + return "Secondary care - Outpatients"; + case "SecondaryCareInPateints": + return "Secondary care - In-patients"; + case "SecondaryCareAmbulance": + return "Secondary care - Ambulance"; + case "SecondaryCareICU": + return "Secondary care - ICU"; + case "PrescribingCommunityPharmacy": + return "Prescribing - Community pharmacy"; + case "PrescribingHospital": + return "Prescribing - Hospital"; + case "PateintReportOutcome": + return "Patient report outcome"; + case "Wearables": + return "Wearables"; + case "LocalAuthority": + return "Local authority"; + case "NationalGovernment": + return "National government"; + case "Community": + return "Community"; + case "Services": + return "Services"; + case "Home": + return "Home"; + case "Private": + return "Private"; + case "SocialCareHealthcareAtHome": + return "Social care - Health care at home"; + case "SocialCareOthersocialData": + return "Social care - Other social data"; + case "Census": + return "Census"; + case "Other": + return "Other"; + } + return null; + } + public HDRDatasetOrigin(Catalogue catalogue) + { + //todo purpose and collectionSource not populating + source = catalogue.DataSource != null?catalogue.DataSource.Split(',').Select(s => mapDataSourceType(s)).Where(s => s != ""):null; + purpose = new List() { mapPurpose(catalogue.Purpose) }; + collectionSource = catalogue.DataSourceSetting != null? catalogue.DataSourceSetting.Split(',').Select(source => mapCollectionSource(source)):null; + datasetType = catalogue.DataType != null ? catalogue.DataType.Split(',').Select(s => mapDatasetType(s)).Where(s => s != "").Select(s => new DataTypeObject() { name = s }) : null; //catalogue.DataType;//this has been + } + + public void Update(Catalogue catalogue) { source = catalogue.DataSource.Split(',').Select(s => mapDataSourceType(s)).Where(s => s != ""); - //collectionSource = catalogue.Source_of_data_collection != null? string.Join(";,", catalogue.Source_of_data_collection.Split(',')) + ';':null; + purpose = new List() { mapPurpose(catalogue.Purpose) }; + collectionSource = catalogue.Source_of_data_collection != null ? catalogue.Source_of_data_collection.Split(',') : null; datasetType = catalogue.DataType != null ? catalogue.DataType.Split(',').Select(s => mapDatasetType(s)).Where(s => s != "").Select(s => new DataTypeObject() { name = s }) : null; //catalogue.DataType;//this has been updated - Console.WriteLine('w'); - //datasetSubType = catalogue.DataSubType != null? string.Join(";,", catalogue.DataSubType.Split(',')) + ';':null; //catalogue.DataSubType; + } - public string purpose { get; set; } + public object purpose { get; set; } public object source { get; set; } - public string collectionSource { get; set; } + public object collectionSource { get; set; } public object datasetType { get; set; }//sometimes string ,somelimes list public string datasetSubType { get; set; } public string imageContrast { get; set; } @@ -275,6 +406,11 @@ public HDRDatasetAccessibility(Catalogue catalogue) access = new HDRDatasetAccess(catalogue); formatAndStandards = new HDRDatasetFormatAndStandards(catalogue); } + + public void Update(Catalogue catalogue) { + if(access is HDRDatasetAccess)((HDRDatasetAccess)access).Update(catalogue); + formatAndStandards.Update(catalogue); + } public object access { get; set; }// there is an issue descerialising this as it's sometimes an object, sometimes a list public HDRDatasetUsage usage { get; set; } public HDRDatasetFormatAndStandards formatAndStandards { get; set; } = new HDRDatasetFormatAndStandards(); @@ -288,6 +424,11 @@ public HDRDatasetProvenance(Catalogue catalogue) temporal = new HDRDatasetTemporal(catalogue); origin = new HDRDatasetOrigin(catalogue); } + + public void Update(Catalogue catalogue) { + temporal.Update(catalogue); + if(origin != null)origin.Update(catalogue); + } public HDRDatasetOrigin origin { get; set; } public HDRDatasetTemporal temporal { get; set; } = new HDRDatasetTemporal(); } @@ -306,6 +447,11 @@ public HDRDatasetCoverage(Catalogue catalogue) { spatial = catalogue.Geographical_coverage; } + + public void Update(Catalogue catalogue) + { + spatial = catalogue.Geographical_coverage; + } public string spatial { get; set; } = "Prussia"; //TODO - min age, max age, coverage, pathway } @@ -338,6 +484,7 @@ public HDRDatasetMetadata(Catalogue catalogue) coverage = new HDRDatasetCoverage(catalogue); accessibility = new HDRDatasetAccessibility(catalogue); } + public HDRDatasetMetadata() { } public string identifier = ""; public string version = "1.0.0"; @@ -350,7 +497,6 @@ public HDRDatasetMetadata() { } public HDRDatasetDocumentation documentation = new HDRDatasetDocumentation(); public HDRDatasetCoverage coverage = new HDRDatasetCoverage(); public HDRDatasetProvenance provenance = new HDRDatasetProvenance(); - //public List tissueSampleCollection = new List { }; public HDRDatasetRequired required { get; set; } } @@ -399,6 +545,8 @@ private class HDRDatasetUpdateMetadata public HDRDatasetAccessibility accessibility { get; set; } public List tissuesSampleCollection = []; + public HDRDatasetDocumentation documentation { get; set; } + public HDRDatasetUpdateMetadata() { } public HDRDatasetUpdateMetadata(HDRDatasetObject datasetObject) { @@ -407,6 +555,17 @@ public HDRDatasetUpdateMetadata(HDRDatasetObject datasetObject) required = datasetObject.latest_metadata.metadata.metadata.required; provenance = datasetObject.latest_metadata.metadata.metadata.provenance; accessibility = datasetObject.latest_metadata.metadata.metadata.accessibility; + documentation = datasetObject.latest_metadata.metadata.metadata.documentation; + } + public void Update(Catalogue catalogue) + { + //todo + summary.Update(catalogue); + coverage.Update(catalogue); + provenance.Update(catalogue); + accessibility.Update(catalogue); + documentation = new HDRDatasetDocumentation(catalogue); + } } @@ -423,7 +582,7 @@ public HDRDatasetUpdateInterMetadata(HDRDatasetObject datasetObject) private class HDRDatasetUpdateBody { - public string status = "DRAFT"; + public string status = "ACTIVE"; public HDRDatasetUpdateInterMetadata metadata { get; set; } public HDRDatasetUpdateBody() { } @@ -432,6 +591,11 @@ public HDRDatasetUpdateBody(HDRDatasetObject datasetObject) metadata = new HDRDatasetUpdateInterMetadata(datasetObject); } + public void Update(Catalogue catalogue) + { + metadata.metadata.Update(catalogue); + } + } public ExecuteCommandExportCataloguesToHDRGateway(IBasicActivateItems activator, string url, int team, string appID, string clientID) @@ -498,14 +662,8 @@ private void UpdateDataset(Catalogue catalogue, HDRDatasetObject existingObject) IncludeFields = true, DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; + body.Update(catalogue); body.metadata.metadata.summary.dataCustodian = null; - body.metadata.metadata.summary.@abstract = "todo123"; - body.metadata.metadata.summary.controlledKeywords = "this;,that;"; - body.metadata.metadata.provenance.temporal.timeLag = "Less than 1 week"; - body.metadata.metadata.provenance.temporal.distributionReleaseDate = null; - body.metadata.metadata.provenance.temporal.accrualPeriodicity = body.metadata.metadata.provenance.temporal.publishingFrequency;//todo check this? - body.metadata.metadata.accessibility.access = new HDRDatasetAccess() { accessRights = "123" }; - body.metadata.metadata.accessibility.usage = null; var jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PatchAsync(url, httpContent)).Result; @@ -530,16 +688,22 @@ public override void Execute() foreach (var catalogue in catalogues) { var existingDataset = CatalogueExists(catalogue); - if (existingDataset != null) - { - //update - UpdateDataset(catalogue, existingDataset); - } - else + if(existingDataset != null) { - //create - CreateDataset(catalogue); + var response = Task.Run(async () => await _client.DeleteAsync($"{_url}/v2/teams/{_team}/datasets/{existingDataset.id}")).Result; + } + CreateDataset(catalogue); + //if (existingDataset != null) + //{ + // //update + // UpdateDataset(catalogue, existingDataset); + //} + //else + //{ + // //create + // CreateDataset(catalogue); + //} } } From 96af888afc480fbeb60f6e9fb5ffd6c526e3e7ea Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 16 Dec 2025 09:54:44 +0000 Subject: [PATCH 06/13] working versioning --- ...cuteCommandExportCataloguesToHDRGateway.cs | 123 +++++++++++------- 1 file changed, 79 insertions(+), 44 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs index 8b55317a3e..f0c486b114 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs @@ -76,7 +76,7 @@ public HDRDatasetMetadataSummary(Catalogue catalogue) { Update(catalogue); } - public void Update(Catalogue catalogue) + public void Update(Catalogue catalogue) { title = catalogue.Name; populationSize = 0; @@ -132,13 +132,14 @@ public HDRDatasetAccess(Catalogue catalogue) { //todo access rights accessRights = " ";// catalogue.Access_options; - jurisdiction = catalogue.Juristiction != null?catalogue.Juristiction.Split(',').ToList():null; + jurisdiction = catalogue.Juristiction != null ? catalogue.Juristiction.Split(',').ToList() : null; deliveryLeadTime = mapUpdateLag(catalogue.UpdateLag); dataController = catalogue.DataController; dataProcessor = catalogue.DataProcessor; } - public void Update(Catalogue catalogue) { + public void Update(Catalogue catalogue) + { jurisdiction = catalogue.Juristiction.Split(',').ToList(); deliveryLeadTime = mapUpdateLag(catalogue.UpdateLag); } @@ -179,12 +180,13 @@ public HDRDatasetTemporal(Catalogue catalogue) } - public void Update(Catalogue catalogue) { + public void Update(Catalogue catalogue) + { timeLag = catalogue.UpdateLag.ToString(); publishingFrequency = catalogue.Update_freq.ToString(); if (catalogue.StartDate != null) startDate = (DateTime)catalogue.StartDate; if (catalogue.EndDate != null) endDate = (DateTime)catalogue.EndDate; - if(catalogue.DatasetReleaseDate != null) distributionReleaseDate = (DateTime)catalogue.DatasetReleaseDate; + if (catalogue.DatasetReleaseDate != null) distributionReleaseDate = (DateTime)catalogue.DatasetReleaseDate; } [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] @@ -196,7 +198,7 @@ public void Update(Catalogue catalogue) { public string publishingFrequency { get; set; }//this needs to be populated public object distributionReleaseDate { get; set; } - public string accrualPeriodicity { get; set; }//this is the same as publishign frequency? + //public string accrualPeriodicity { get; set; }//this is the same as publishign frequency? } public class HDRDatasetFormatAndStandards @@ -282,7 +284,7 @@ private string mapPurpose(Catalogue.DatasetPurpose purpose) case Catalogue.DatasetPurpose.ResearchCohort: - return "Research cohort"; + return "Research cohort"; case Catalogue.DatasetPurpose.Study: return "Study"; case Catalogue.DatasetPurpose.DiseaseRegistry: @@ -301,8 +303,8 @@ private string mapPurpose(Catalogue.DatasetPurpose purpose) return "Statutory"; case Catalogue.DatasetPurpose.Other: return "Other"; - } - return null; + } + return null; } private class DataTypeObject @@ -313,9 +315,10 @@ private class DataTypeObject private string mapCollectionSource(string source) { - switch (source) { + switch (source) + { case "CohortStudyTrial": - return "Cohort, study, trial"; + return "Cohort, study, trial"; case "Clininc": return "Clinic"; case "PrimaryCareReferrals": @@ -362,25 +365,25 @@ private string mapCollectionSource(string source) return "Census"; case "Other": return "Other"; - } + } return null; } public HDRDatasetOrigin(Catalogue catalogue) { //todo purpose and collectionSource not populating - source = catalogue.DataSource != null?catalogue.DataSource.Split(',').Select(s => mapDataSourceType(s)).Where(s => s != ""):null; + source = catalogue.DataSource != null ? catalogue.DataSource.Split(',').Select(s => mapDataSourceType(s)).Where(s => s != "") : null; purpose = new List() { mapPurpose(catalogue.Purpose) }; - collectionSource = catalogue.DataSourceSetting != null? catalogue.DataSourceSetting.Split(',').Select(source => mapCollectionSource(source)):null; + collectionSource = catalogue.DataSourceSetting != null ? catalogue.DataSourceSetting.Split(',').Select(source => mapCollectionSource(source)) : null; datasetType = catalogue.DataType != null ? catalogue.DataType.Split(',').Select(s => mapDatasetType(s)).Where(s => s != "").Select(s => new DataTypeObject() { name = s }) : null; //catalogue.DataType;//this has been } public void Update(Catalogue catalogue) { - source = catalogue.DataSource.Split(',').Select(s => mapDataSourceType(s)).Where(s => s != ""); + source = catalogue.DataSource != null ? catalogue.DataSource.Split(',').Select(s => mapDataSourceType(s)).Where(s => s != "") : null; purpose = new List() { mapPurpose(catalogue.Purpose) }; - collectionSource = catalogue.Source_of_data_collection != null ? catalogue.Source_of_data_collection.Split(',') : null; - datasetType = catalogue.DataType != null ? catalogue.DataType.Split(',').Select(s => mapDatasetType(s)).Where(s => s != "").Select(s => new DataTypeObject() { name = s }) : null; //catalogue.DataType;//this has been updated + collectionSource = catalogue.DataSourceSetting != null ? catalogue.DataSourceSetting.Split(',').Select(source => mapCollectionSource(source)) : null; + datasetType = catalogue.DataType != null ? catalogue.DataType.Split(',').Select(s => mapDatasetType(s)).Where(s => s != "").Select(s => new DataTypeObject() { name = s }) : null; //catalogue.DataType;//this has been } public object purpose { get; set; } @@ -393,7 +396,7 @@ public void Update(Catalogue catalogue) public class HDRDatasetUsage { - public List dataUseLimitation { get; set; } + public object dataUseLimitation { get; set; } public object resourceCreator { get; set; } public List dataUseRequirements { get; set; } = []; } @@ -407,8 +410,9 @@ public HDRDatasetAccessibility(Catalogue catalogue) formatAndStandards = new HDRDatasetFormatAndStandards(catalogue); } - public void Update(Catalogue catalogue) { - if(access is HDRDatasetAccess)((HDRDatasetAccess)access).Update(catalogue); + public void Update(Catalogue catalogue) + { + if (access is HDRDatasetAccess) ((HDRDatasetAccess)access).Update(catalogue); formatAndStandards.Update(catalogue); } public object access { get; set; }// there is an issue descerialising this as it's sometimes an object, sometimes a list @@ -425,9 +429,10 @@ public HDRDatasetProvenance(Catalogue catalogue) origin = new HDRDatasetOrigin(catalogue); } - public void Update(Catalogue catalogue) { + public void Update(Catalogue catalogue) + { temporal.Update(catalogue); - if(origin != null)origin.Update(catalogue); + if (origin != null) origin.Update(catalogue); } public HDRDatasetOrigin origin { get; set; } public HDRDatasetTemporal temporal { get; set; } = new HDRDatasetTemporal(); @@ -535,27 +540,42 @@ public HDRDatasetCreationBody(Catalogue catalogue) } } + private class HDRDatasetUpdateMetadata { + public string version = "1.0.0"; + public DateTime issued = DateTime.Now; + public DateTime modified = DateTime.Now; + public string identifier = Guid.NewGuid().ToString(); public HDRDatasetMetadataSummary summary { get; set; } public HDRDatasetCoverage coverage { get; set; } - public HDRDatasetRequired required { get; set; } + public List revisions = new(); public HDRDatasetProvenance provenance { get; set; } public List observations = []; public HDRDatasetAccessibility accessibility { get; set; } - public List tissuesSampleCollection = []; public HDRDatasetDocumentation documentation { get; set; } public HDRDatasetUpdateMetadata() { } + + private string UpdateSemVer(string currentVersion) + { + var ver = currentVersion.Split('.'); + var newPatch = Int32.Parse(ver.Last()) + 1; + ver[2] = newPatch.ToString(); + return String.Join('.', ver); + + } + public HDRDatasetUpdateMetadata(HDRDatasetObject datasetObject) { summary = datasetObject.latest_metadata.metadata.metadata.summary; coverage = datasetObject.latest_metadata.metadata.metadata.coverage; - required = datasetObject.latest_metadata.metadata.metadata.required; + //required = datasetObject.latest_metadata.metadata.metadata.required; provenance = datasetObject.latest_metadata.metadata.metadata.provenance; accessibility = datasetObject.latest_metadata.metadata.metadata.accessibility; documentation = datasetObject.latest_metadata.metadata.metadata.documentation; + version = UpdateSemVer(datasetObject.latest_metadata.metadata.metadata.required.version); } public void Update(Catalogue catalogue) { @@ -574,10 +594,13 @@ private class HDRDatasetUpdateInterMetadata public HDRDatasetUpdateInterMetadata(HDRDatasetObject datasetObject) { + metadata = new HDRDatasetUpdateMetadata(datasetObject); } public HDRDatasetUpdateMetadata metadata { get; set; } + public string schemaModel = "HDRUK"; + public string schemaVersion = "4.0.0"; } private class HDRDatasetUpdateBody @@ -663,17 +686,29 @@ private void UpdateDataset(Catalogue catalogue, HDRDatasetObject existingObject) DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; body.Update(catalogue); - body.metadata.metadata.summary.dataCustodian = null; + body.metadata.metadata.accessibility.usage.dataUseLimitation = new List() { "General research use" }; + body.metadata.metadata.accessibility.usage.resourceCreator = null; + body.metadata.metadata.accessibility.access = new HDRDatasetAccess(catalogue) + { + accessRights = "null" + }; + body.metadata.metadata.accessibility.formatAndStandards.conformsTo = new List() { "LOCAL" }; var jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); - var response = Task.Run(async () => await _client.PatchAsync(url, httpContent)).Result; + var response = Task.Run(async () => await _client.PutAsync(url, httpContent)).Result; if (response.StatusCode != System.Net.HttpStatusCode.OK) { var contentString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; - Console.WriteLine($"unable to update Dataset '{catalogue.Name}': {contentString}"); - + Console.WriteLine($"unable to update Dataset '{catalogue.Name}': {contentString}. Will attempt to update in draft."); + body.status = "DRAFT"; + jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); + httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + response = Task.Run(async () => await _client.PutAsync(url, httpContent)).Result; + } + else + { + Console.WriteLine($"Updated Dataset '{catalogue.Name}'"); } - Console.WriteLine($"Updated Dataset '{catalogue.Name}'"); } public override void Execute() @@ -688,22 +723,22 @@ public override void Execute() foreach (var catalogue in catalogues) { var existingDataset = CatalogueExists(catalogue); - if(existingDataset != null) - { - var response = Task.Run(async () => await _client.DeleteAsync($"{_url}/v2/teams/{_team}/datasets/{existingDataset.id}")).Result; - - } - CreateDataset(catalogue); - //if (existingDataset != null) - //{ - // //update - // UpdateDataset(catalogue, existingDataset); - //} - //else + //if(existingDataset != null) //{ - // //create - // CreateDataset(catalogue); + // var response = Task.Run(async () => await _client.DeleteAsync($"{_url}/v2/teams/{_team}/datasets/{existingDataset.id}")).Result; + //} + //CreateDataset(catalogue); + if (existingDataset != null) + { + //update + UpdateDataset(catalogue, existingDataset); + } + else + { + //create + CreateDataset(catalogue); + } } } From 16a19f961d7d373e169a1718ce4b1c11edeb5ac0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 16 Dec 2025 11:58:44 +0000 Subject: [PATCH 07/13] working update --- ...cuteCommandExportCataloguesToHDRGateway.cs | 102 +++++++----------- 1 file changed, 37 insertions(+), 65 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs index f0c486b114..7b78f0d51a 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs @@ -1,26 +1,14 @@ -using Amazon.Runtime.Internal.Endpoints.StandardLibrary; -using Amazon.SecurityToken.Model; -using Azure; -using MathNet.Numerics; + using MongoDB.Driver; using Newtonsoft.Json; -using NPOI.OpenXmlFormats.Dml; -using NPOI.POIFS.Crypt; -using NPOI.SS.UserModel.Charts; -using NPOI.XWPF.UserModel; using Rdmp.Core.Curation.Data; -using Rdmp.Core.Dataset.Confluence; -using Rdmp.Core.Repositories; -using Renci.SshNet.Sftp; using System; using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Net.Http; -using System.Net.Http.Json; using System.Text; using System.Text.Json; -using System.Text.RegularExpressions; using System.Threading.Tasks; namespace Rdmp.Core.CommandExecution.AtomicCommands @@ -47,9 +35,9 @@ public HDRDatasetDataCustodian(Catalogue catalogue) name = catalogue.DataProcessor; contactPoint = catalogue.Administrative_contact_email; } - public string name { get; set; }//this needs to be populated + public string name { get; set; } public int identifier { get; set; } = 0; - public string contactPoint { get; set; }//this needs to be populated + public string contactPoint { get; set; } public object logo { get; set; } public object description { get; set; } public object memberOf { get; set; } @@ -68,7 +56,7 @@ public void Update(Catalogue catalogue) description = catalogue.Description; } - public string description { get; set; }//this needs to be populated; + public string description { get; set; } } private class HDRDatasetMetadataSummary { @@ -79,7 +67,7 @@ public HDRDatasetMetadataSummary(Catalogue catalogue) public void Update(Catalogue catalogue) { title = catalogue.Name; - populationSize = 0; + populationSize = 0;//todo @abstract = catalogue.ShortDescription; contactPoint = catalogue.Administrative_contact_email; dataCustodian = new HDRDatasetDataCustodian(catalogue); @@ -88,8 +76,8 @@ public void Update(Catalogue catalogue) } public HDRDatasetMetadataSummary() { } - public string @abstract { get; set; }//this needs to be populated - public string contactPoint { get; set; }//this needs to be populated + public string @abstract { get; set; } + public string contactPoint { get; set; } public object keywords { get; set; } public object doiName { get; set; } public string title { get; set; } @@ -97,11 +85,12 @@ public HDRDatasetMetadataSummary() { } public int populationSize { get; set; } public object alternateIdentifiers { get; set; } public object controlledKeywords { get; set; } + //public string inPipeline = "Available"; } private class HDRDatasetAccess { - private string mapUpdateLag(Catalogue.UpdateLagTimes time) + private static string MapUpdateLag(Catalogue.UpdateLagTimes time) { switch (time) { @@ -130,10 +119,8 @@ private string mapUpdateLag(Catalogue.UpdateLagTimes time) public HDRDatasetAccess() { } public HDRDatasetAccess(Catalogue catalogue) { - //todo access rights - accessRights = " ";// catalogue.Access_options; jurisdiction = catalogue.Juristiction != null ? catalogue.Juristiction.Split(',').ToList() : null; - deliveryLeadTime = mapUpdateLag(catalogue.UpdateLag); + deliveryLeadTime = MapUpdateLag(catalogue.UpdateLag); dataController = catalogue.DataController; dataProcessor = catalogue.DataProcessor; } @@ -141,17 +128,17 @@ public HDRDatasetAccess(Catalogue catalogue) public void Update(Catalogue catalogue) { jurisdiction = catalogue.Juristiction.Split(',').ToList(); - deliveryLeadTime = mapUpdateLag(catalogue.UpdateLag); + deliveryLeadTime = MapUpdateLag(catalogue.UpdateLag); } public string deliveryLeadTime { get; set; } public object jurisdiction { get; set; } public string dataController { get; set; } public string dataProcessor { get; set; } - public string accessRights { get; set; }//this needs to be populated - public string accessService { get; set; } + public string accessRights { get; set; } = "https://www.dundee.ac.uk/hic/governance-service"; + public string accessService { get; set; } = "TODO this is some desc"; public string accessRequestCost { get; set; } - public object accessServiceCategory { get; set; } + public object accessServiceCategory { get; set; } = "TRE/SDE"; } private class CustomDateTimeConverterThreeMilliseconds : JsonConverter @@ -193,12 +180,10 @@ public void Update(Catalogue catalogue) public DateTime? endDate { get; set; } [JsonConverter(typeof(CustomDateTimeConverterThreeMilliseconds))] - public DateTime startDate { get; set; }// = DateTime.Now; - public string timeLag { get; set; }//this needs to be populated - public string publishingFrequency { get; set; }//this needs to be populated + public DateTime startDate { get; set; } + public string timeLag { get; set; } + public string publishingFrequency { get; set; } public object distributionReleaseDate { get; set; } - - //public string accrualPeriodicity { get; set; }//this is the same as publishign frequency? } public class HDRDatasetFormatAndStandards @@ -225,7 +210,7 @@ public class HDRDatasetOrigin public HDRDatasetOrigin() { } - private string mapDatasetType(string datasetType) + private static string MapDatasetType(string datasetType) { switch (datasetType) { @@ -255,7 +240,7 @@ private string mapDatasetType(string datasetType) return ""; } - private string mapDataSourceType(string dataSourceType) + private static string MapDataSourceType(string dataSourceType) { switch (dataSourceType) { @@ -277,7 +262,7 @@ private string mapDataSourceType(string dataSourceType) return ""; } - private string mapPurpose(Catalogue.DatasetPurpose purpose) + private static string MapPurpose(Catalogue.DatasetPurpose purpose) { switch (purpose) { @@ -313,7 +298,7 @@ private class DataTypeObject public List subTypes = new(); } - private string mapCollectionSource(string source) + private static string MapCollectionSource(string source) { switch (source) { @@ -371,34 +356,30 @@ private string mapCollectionSource(string source) public HDRDatasetOrigin(Catalogue catalogue) { - //todo purpose and collectionSource not populating - source = catalogue.DataSource != null ? catalogue.DataSource.Split(',').Select(s => mapDataSourceType(s)).Where(s => s != "") : null; - purpose = new List() { mapPurpose(catalogue.Purpose) }; - collectionSource = catalogue.DataSourceSetting != null ? catalogue.DataSourceSetting.Split(',').Select(source => mapCollectionSource(source)) : null; - datasetType = catalogue.DataType != null ? catalogue.DataType.Split(',').Select(s => mapDatasetType(s)).Where(s => s != "").Select(s => new DataTypeObject() { name = s }) : null; //catalogue.DataType;//this has been + Update(catalogue); } public void Update(Catalogue catalogue) { - source = catalogue.DataSource != null ? catalogue.DataSource.Split(',').Select(s => mapDataSourceType(s)).Where(s => s != "") : null; - purpose = new List() { mapPurpose(catalogue.Purpose) }; - collectionSource = catalogue.DataSourceSetting != null ? catalogue.DataSourceSetting.Split(',').Select(source => mapCollectionSource(source)) : null; - datasetType = catalogue.DataType != null ? catalogue.DataType.Split(',').Select(s => mapDatasetType(s)).Where(s => s != "").Select(s => new DataTypeObject() { name = s }) : null; //catalogue.DataType;//this has been + source = catalogue.DataSource != null ? catalogue.DataSource.Split(',').Select(s => MapDataSourceType(s)).Where(s => s != "") : null; + purpose = new List() { MapPurpose(catalogue.Purpose) }; + collectionSource = catalogue.DataSourceSetting != null ? catalogue.DataSourceSetting.Split(',').Select(source => MapCollectionSource(source)) : null; + datasetType = catalogue.DataType != null ? catalogue.DataType.Split(',').Select(s => MapDatasetType(s)).Where(s => s != "").Select(s => new DataTypeObject() { name = s }) : null; } public object purpose { get; set; } public object source { get; set; } public object collectionSource { get; set; } - public object datasetType { get; set; }//sometimes string ,somelimes list + public object datasetType { get; set; } public string datasetSubType { get; set; } public string imageContrast { get; set; } } public class HDRDatasetUsage { - public object dataUseLimitation { get; set; } + public object dataUseLimitation = new List() { "General research use" }; public object resourceCreator { get; set; } - public List dataUseRequirements { get; set; } = []; + public List dataUseRequirements = ["Disclosure control"]; } private class HDRDatasetAccessibility { @@ -408,6 +389,8 @@ public HDRDatasetAccessibility(Catalogue catalogue) { access = new HDRDatasetAccess(catalogue); formatAndStandards = new HDRDatasetFormatAndStandards(catalogue); + usage = new HDRDatasetUsage(); + usage.resourceCreator = "This is a citation"; } public void Update(Catalogue catalogue) @@ -415,8 +398,8 @@ public void Update(Catalogue catalogue) if (access is HDRDatasetAccess) ((HDRDatasetAccess)access).Update(catalogue); formatAndStandards.Update(catalogue); } - public object access { get; set; }// there is an issue descerialising this as it's sometimes an object, sometimes a list - public HDRDatasetUsage usage { get; set; } + public object access { get; set; } + public HDRDatasetUsage usage { get; set; } = new(); public HDRDatasetFormatAndStandards formatAndStandards { get; set; } = new HDRDatasetFormatAndStandards(); } private class HDRDatasetProvenance @@ -457,7 +440,7 @@ public void Update(Catalogue catalogue) { spatial = catalogue.Geographical_coverage; } - public string spatial { get; set; } = "Prussia"; + public string spatial { get; set; } //TODO - min age, max age, coverage, pathway } @@ -480,9 +463,6 @@ private class HDRDatasetMetadata public HDRDatasetMetadata(Catalogue catalogue) { - version = "1.0.0"; - modified = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); - issued = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); summary = new HDRDatasetMetadataSummary(catalogue); documentation = new HDRDatasetDocumentation(catalogue); provenance = new HDRDatasetProvenance(catalogue); @@ -494,8 +474,8 @@ public HDRDatasetMetadata() { } public string identifier = ""; public string version = "1.0.0"; public List revisions = new List(); - public string modified; - public string issued; + public string modified= DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); + public string issued = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); public HDRDatasetMetadataSummary summary = new HDRDatasetMetadataSummary(); public HDRDatasetAccessibility accessibility = new HDRDatasetAccessibility(); public List observations = new List(); @@ -571,7 +551,6 @@ public HDRDatasetUpdateMetadata(HDRDatasetObject datasetObject) { summary = datasetObject.latest_metadata.metadata.metadata.summary; coverage = datasetObject.latest_metadata.metadata.metadata.coverage; - //required = datasetObject.latest_metadata.metadata.metadata.required; provenance = datasetObject.latest_metadata.metadata.metadata.provenance; accessibility = datasetObject.latest_metadata.metadata.metadata.accessibility; documentation = datasetObject.latest_metadata.metadata.metadata.documentation; @@ -579,13 +558,11 @@ public HDRDatasetUpdateMetadata(HDRDatasetObject datasetObject) } public void Update(Catalogue catalogue) { - //todo summary.Update(catalogue); coverage.Update(catalogue); provenance.Update(catalogue); accessibility.Update(catalogue); documentation = new HDRDatasetDocumentation(catalogue); - } } @@ -686,6 +663,7 @@ private void UpdateDataset(Catalogue catalogue, HDRDatasetObject existingObject) DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; body.Update(catalogue); + //todo this needs improved body.metadata.metadata.accessibility.usage.dataUseLimitation = new List() { "General research use" }; body.metadata.metadata.accessibility.usage.resourceCreator = null; body.metadata.metadata.accessibility.access = new HDRDatasetAccess(catalogue) @@ -723,12 +701,6 @@ public override void Execute() foreach (var catalogue in catalogues) { var existingDataset = CatalogueExists(catalogue); - //if(existingDataset != null) - //{ - // var response = Task.Run(async () => await _client.DeleteAsync($"{_url}/v2/teams/{_team}/datasets/{existingDataset.id}")).Result; - - //} - //CreateDataset(catalogue); if (existingDataset != null) { //update From ff65ed6083e6cbf1d5dabc76cb42def89a9fda6f Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 16 Dec 2025 13:40:21 +0000 Subject: [PATCH 08/13] working update --- ...cuteCommandExportCataloguesToHDRGateway.cs | 155 +++++++++++------- 1 file changed, 98 insertions(+), 57 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs index 7b78f0d51a..c7b34df2fc 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO; using System.Linq; using System.Net.Http; using System.Text; @@ -26,6 +27,35 @@ public class ExecuteCommandExportCataloguesToHDRGateway : BasicCommandExecution, private readonly string _clientID; private readonly HttpClient _client = new(); private readonly string _datasetsEndpoint = "/v2/datasets"; + private InputJSONDetails _inputJsonDetails = new(); + private class InputJSONDetails + { + /// + /// { + /// "accessRights":"https://www.dundee.ac.uk/hic/governance-service", + /// "accessService":"some kind of access message- get in touch", + /// "accessServiceCategory":"TRE/SDE", + /// "conformsTo":["LOCAL"], + /// "vocabularyEncodingScheme":["LOCAL"], + /// "language":["en"], + /// "format":["CSV","Database"] + /// "dataUseLimitation":["General research use"], + /// "resourceCreator":"Please cite us!", + /// "dataUseRequirements":["Disclosure control"] + /// } + /// + public string accessRights { get; set; } + public string accessService { get; set; } + public string accessRequestCost { get; set; } + public string accessServiceCategory { get; set; } + public List conformsTo { get; set; }// = new List() { "LOCAL" }; + public List vocabularyEncodingScheme { get; set; }// = new List() { "LOCAL" }; + public List language { get; set; }// = new List() { "en" }; + public List format { get; set; }// = new List() { "CSV", "Database" }; + public List dataUseLimitation { get; set; } + public string resourceCreator { get; set; } + public List dataUseRequirements { get; set; } + } public class HDRDatasetDataCustodian { @@ -67,7 +97,7 @@ public HDRDatasetMetadataSummary(Catalogue catalogue) public void Update(Catalogue catalogue) { title = catalogue.Name; - populationSize = 0;//todo + populationSize = 0; @abstract = catalogue.ShortDescription; contactPoint = catalogue.Administrative_contact_email; dataCustodian = new HDRDatasetDataCustodian(catalogue); @@ -85,7 +115,6 @@ public HDRDatasetMetadataSummary() { } public int populationSize { get; set; } public object alternateIdentifiers { get; set; } public object controlledKeywords { get; set; } - //public string inPipeline = "Available"; } private class HDRDatasetAccess { @@ -117,28 +146,30 @@ private static string MapUpdateLag(Catalogue.UpdateLagTimes time) } public HDRDatasetAccess() { } - public HDRDatasetAccess(Catalogue catalogue) + public HDRDatasetAccess(Catalogue catalogue, InputJSONDetails details) { - jurisdiction = catalogue.Juristiction != null ? catalogue.Juristiction.Split(',').ToList() : null; - deliveryLeadTime = MapUpdateLag(catalogue.UpdateLag); - dataController = catalogue.DataController; - dataProcessor = catalogue.DataProcessor; + Update(catalogue, details); } - public void Update(Catalogue catalogue) + public void Update(Catalogue catalogue, InputJSONDetails details) { - jurisdiction = catalogue.Juristiction.Split(',').ToList(); + jurisdiction = catalogue.Juristiction != null ? catalogue.Juristiction.Split(',').ToList() : null; deliveryLeadTime = MapUpdateLag(catalogue.UpdateLag); + dataController = catalogue.DataController; + dataProcessor = catalogue.DataProcessor; + accessRights = details.accessRights; + accessService = details.accessService; + accessServiceCategory = details.accessServiceCategory; } public string deliveryLeadTime { get; set; } public object jurisdiction { get; set; } public string dataController { get; set; } public string dataProcessor { get; set; } - public string accessRights { get; set; } = "https://www.dundee.ac.uk/hic/governance-service"; - public string accessService { get; set; } = "TODO this is some desc"; + public string accessRights { get; set; } + public string accessService { get; set; } public string accessRequestCost { get; set; } - public object accessServiceCategory { get; set; } = "TRE/SDE"; + public object accessServiceCategory { get; set; } } private class CustomDateTimeConverterThreeMilliseconds : JsonConverter @@ -186,21 +217,24 @@ public void Update(Catalogue catalogue) public object distributionReleaseDate { get; set; } } - public class HDRDatasetFormatAndStandards + private class HDRDatasetFormatAndStandards { public HDRDatasetFormatAndStandards() { } - public HDRDatasetFormatAndStandards(Catalogue catalogue) + public HDRDatasetFormatAndStandards(Catalogue catalogue, InputJSONDetails details) { - //todo + Update(catalogue, details); } - public void Update(Catalogue catalogue) + public void Update(Catalogue catalogue, InputJSONDetails details) { - //todo + conformsTo = details.conformsTo; + vocabularyEncodingScheme = details.vocabularyEncodingScheme; + language = details.language; + format = details.format; } - public object conformsTo { get; set; } = new List() { "LOCAL" }; - public object vocabularyEncodingScheme { get; set; } = new List() { "LOCAL" }; - public object language { get; set; } = new List() { "en" }; - public object format { get; set; } = new List() { "CSV", "Database" }; + public object conformsTo { get; set; } + public object vocabularyEncodingScheme { get; set; } + public object language { get; set; } + public object format { get; set; } } public class HDRDatasetOrigin @@ -255,7 +289,7 @@ private static string MapDataSourceType(string dataSourceType) case "PaperBased": return "Paper-based"; case "FreeTextNLP": - return "TODO"; + return "Free text NLP"; case "MachineLearning": return "Machine generated"; } @@ -375,31 +409,40 @@ public void Update(Catalogue catalogue) public string imageContrast { get; set; } } - public class HDRDatasetUsage + private class HDRDatasetUsage { - public object dataUseLimitation = new List() { "General research use" }; + public HDRDatasetUsage() { } + public HDRDatasetUsage(InputJSONDetails details) + { + dataUseLimitation = details.dataUseLimitation; + resourceCreator = details.resourceCreator; + dataUseRequirements = details.dataUseRequirements; + } + + public object dataUseLimitation { get; set; } public object resourceCreator { get; set; } - public List dataUseRequirements = ["Disclosure control"]; + public List dataUseRequirements { get; set; } } private class HDRDatasetAccessibility { public HDRDatasetAccessibility() { } - public HDRDatasetAccessibility(Catalogue catalogue) + public HDRDatasetAccessibility(Catalogue catalogue, InputJSONDetails details) { - access = new HDRDatasetAccess(catalogue); - formatAndStandards = new HDRDatasetFormatAndStandards(catalogue); - usage = new HDRDatasetUsage(); - usage.resourceCreator = "This is a citation"; + access = new HDRDatasetAccess(catalogue, details); + formatAndStandards = new HDRDatasetFormatAndStandards(catalogue, details); + usage = new HDRDatasetUsage(details); } - public void Update(Catalogue catalogue) + public void Update(Catalogue catalogue, InputJSONDetails details) { - if (access is HDRDatasetAccess) ((HDRDatasetAccess)access).Update(catalogue); - formatAndStandards.Update(catalogue); + //if (access is HDRDatasetAccess) ((HDRDatasetAccess)access).Update(catalogue, details); + access = new HDRDatasetAccess(catalogue, details); + formatAndStandards.Update(catalogue, details); + usage = new HDRDatasetUsage(details); } public object access { get; set; } - public HDRDatasetUsage usage { get; set; } = new(); + public HDRDatasetUsage usage { get; set; } public HDRDatasetFormatAndStandards formatAndStandards { get; set; } = new HDRDatasetFormatAndStandards(); } private class HDRDatasetProvenance @@ -441,7 +484,6 @@ public void Update(Catalogue catalogue) spatial = catalogue.Geographical_coverage; } public string spatial { get; set; } - //TODO - min age, max age, coverage, pathway } private class HDRDatasetRequired @@ -461,20 +503,20 @@ public HDRDatasetRequired() { } private class HDRDatasetMetadata { - public HDRDatasetMetadata(Catalogue catalogue) + public HDRDatasetMetadata(Catalogue catalogue, InputJSONDetails details) { summary = new HDRDatasetMetadataSummary(catalogue); documentation = new HDRDatasetDocumentation(catalogue); provenance = new HDRDatasetProvenance(catalogue); coverage = new HDRDatasetCoverage(catalogue); - accessibility = new HDRDatasetAccessibility(catalogue); + accessibility = new HDRDatasetAccessibility(catalogue, details); } public HDRDatasetMetadata() { } public string identifier = ""; public string version = "1.0.0"; public List revisions = new List(); - public string modified= DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); + public string modified = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); public string issued = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); public HDRDatasetMetadataSummary summary = new HDRDatasetMetadataSummary(); public HDRDatasetAccessibility accessibility = new HDRDatasetAccessibility(); @@ -514,9 +556,9 @@ private class HDRDatasetCreationBody public string status = "ACTIVE"; public string create_origin = "MANUAL"; public HDRDatasetMetadata metadata = new(); - public HDRDatasetCreationBody(Catalogue catalogue) + public HDRDatasetCreationBody(Catalogue catalogue, InputJSONDetails details) { - metadata = new HDRDatasetMetadata(catalogue); + metadata = new HDRDatasetMetadata(catalogue, details); } } @@ -556,12 +598,12 @@ public HDRDatasetUpdateMetadata(HDRDatasetObject datasetObject) documentation = datasetObject.latest_metadata.metadata.metadata.documentation; version = UpdateSemVer(datasetObject.latest_metadata.metadata.metadata.required.version); } - public void Update(Catalogue catalogue) + public void Update(Catalogue catalogue, InputJSONDetails details) { summary.Update(catalogue); coverage.Update(catalogue); provenance.Update(catalogue); - accessibility.Update(catalogue); + accessibility.Update(catalogue,details); documentation = new HDRDatasetDocumentation(catalogue); } } @@ -591,20 +633,22 @@ public HDRDatasetUpdateBody(HDRDatasetObject datasetObject) metadata = new HDRDatasetUpdateInterMetadata(datasetObject); } - public void Update(Catalogue catalogue) + public void Update(Catalogue catalogue, InputJSONDetails details) { - metadata.metadata.Update(catalogue); + metadata.metadata.Update(catalogue, details); } } - public ExecuteCommandExportCataloguesToHDRGateway(IBasicActivateItems activator, string url, int team, string appID, string clientID) + public ExecuteCommandExportCataloguesToHDRGateway(IBasicActivateItems activator, string url, int team, string appID, string clientID, FileInfo configFile) { _activator = activator; _url = url.TrimEnd('/'); _team = team; _appID = appID; _clientID = clientID; + var x = File.ReadAllText(configFile.Name); + _inputJsonDetails = JsonConvert.DeserializeObject(File.ReadAllText(configFile.Name)); } private HDRDatasetObject CatalogueExists(Catalogue catalogue) @@ -623,7 +667,7 @@ private HDRDatasetObject CatalogueExists(Catalogue catalogue) private void CreateDataset(Catalogue catalogue) { var url = $"{_url}/v2/teams/{_team}/datasets?input_schema=HDRUK&input_version=4.0.0?input_schema=HDRUK&input_version=4.0.0"; - HDRDatasetCreationBody body = new HDRDatasetCreationBody(catalogue); + HDRDatasetCreationBody body = new HDRDatasetCreationBody(catalogue,_inputJsonDetails); var serializeOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -662,15 +706,14 @@ private void UpdateDataset(Catalogue catalogue, HDRDatasetObject existingObject) IncludeFields = true, DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; - body.Update(catalogue); - //todo this needs improved - body.metadata.metadata.accessibility.usage.dataUseLimitation = new List() { "General research use" }; - body.metadata.metadata.accessibility.usage.resourceCreator = null; - body.metadata.metadata.accessibility.access = new HDRDatasetAccess(catalogue) - { - accessRights = "null" - }; - body.metadata.metadata.accessibility.formatAndStandards.conformsTo = new List() { "LOCAL" }; + body.Update(catalogue,_inputJsonDetails); + //body.metadata.metadata.accessibility.usage.dataUseLimitation = new List() { "General research use" }; + //body.metadata.metadata.accessibility.usage.resourceCreator = "Ciration!"; + //body.metadata.metadata.accessibility.access = new HDRDatasetAccess(catalogue,_inputJsonDetails) + //{ + //accessRights = "null" + //}; + //body.metadata.metadata.accessibility.formatAndStandards.conformsTo = new List() { "LOCAL" }; var jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PutAsync(url, httpContent)).Result; @@ -703,12 +746,10 @@ public override void Execute() var existingDataset = CatalogueExists(catalogue); if (existingDataset != null) { - //update UpdateDataset(catalogue, existingDataset); } else { - //create CreateDataset(catalogue); } } From e3d828235db971bbd402ee82fd3254c63a6e62e4 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 16 Dec 2025 14:12:25 +0000 Subject: [PATCH 09/13] update ignore list --- CHANGELOG.md | 3 +++ ...ExecuteCommandExportCataloguesToHDRGateway.cs | 16 ++++------------ .../EvaluateNamespacesAndSolutionFoldersTests.cs | 3 ++- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7242d3fff..b86253fd82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.1.2] - Unreleased +- Add CLI Command to export Catalogue Info to HDR Gateway (see [Documentation\Catalogues\ExportCataloguesToHDR.md]) + ## [9.1.1] - 2025-12-02 - Allow Atlassian service workers to write to Confluence from RDMP diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs index c7b34df2fc..95f6ca3071 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs @@ -48,10 +48,10 @@ private class InputJSONDetails public string accessService { get; set; } public string accessRequestCost { get; set; } public string accessServiceCategory { get; set; } - public List conformsTo { get; set; }// = new List() { "LOCAL" }; - public List vocabularyEncodingScheme { get; set; }// = new List() { "LOCAL" }; - public List language { get; set; }// = new List() { "en" }; - public List format { get; set; }// = new List() { "CSV", "Database" }; + public List conformsTo { get; set; } + public List vocabularyEncodingScheme { get; set; } + public List language { get; set; } + public List format { get; set; } public List dataUseLimitation { get; set; } public string resourceCreator { get; set; } public List dataUseRequirements { get; set; } @@ -436,7 +436,6 @@ public HDRDatasetAccessibility(Catalogue catalogue, InputJSONDetails details) public void Update(Catalogue catalogue, InputJSONDetails details) { - //if (access is HDRDatasetAccess) ((HDRDatasetAccess)access).Update(catalogue, details); access = new HDRDatasetAccess(catalogue, details); formatAndStandards.Update(catalogue, details); usage = new HDRDatasetUsage(details); @@ -707,13 +706,6 @@ private void UpdateDataset(Catalogue catalogue, HDRDatasetObject existingObject) DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; body.Update(catalogue,_inputJsonDetails); - //body.metadata.metadata.accessibility.usage.dataUseLimitation = new List() { "General research use" }; - //body.metadata.metadata.accessibility.usage.resourceCreator = "Ciration!"; - //body.metadata.metadata.accessibility.access = new HDRDatasetAccess(catalogue,_inputJsonDetails) - //{ - //accessRights = "null" - //}; - //body.metadata.metadata.accessibility.formatAndStandards.conformsTo = new List() { "LOCAL" }; var jsonString = System.Text.Json.JsonSerializer.Serialize(body, serializeOptions); var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); var response = Task.Run(async () => await _client.PutAsync(url, httpContent)).Result; diff --git a/Rdmp.UI.Tests/DesignPatternTests/EvaluateNamespacesAndSolutionFoldersTests.cs b/Rdmp.UI.Tests/DesignPatternTests/EvaluateNamespacesAndSolutionFoldersTests.cs index 69bb58594f..0539d93c37 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/EvaluateNamespacesAndSolutionFoldersTests.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/EvaluateNamespacesAndSolutionFoldersTests.cs @@ -34,7 +34,8 @@ public class EvaluateNamespacesAndSolutionFoldersTests : DatabaseTests "ProjectInstaller.cs", "ProjectInstaller.Designer.cs", "TableView.cs", - "TreeView.cs" + "TreeView.cs", + "ExecuteCommandExportCataloguesToHDRGateway.cs" }; [Test] From 147466ad3004e40b012c9095039c1d83ea134495 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 16 Dec 2025 14:27:09 +0000 Subject: [PATCH 10/13] add to correct ignore list --- Rdmp.UI.Tests/DesignPatternTests/CsProjFileTidy.cs | 2 +- .../EvaluateNamespacesAndSolutionFoldersTests.cs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Rdmp.UI.Tests/DesignPatternTests/CsProjFileTidy.cs b/Rdmp.UI.Tests/DesignPatternTests/CsProjFileTidy.cs index 083340d6c3..3311ca710e 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/CsProjFileTidy.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/CsProjFileTidy.cs @@ -28,7 +28,7 @@ internal class CsProjFileTidy "Attributes.cs", "AssemblyInfo.cs", "Annotations.cs", "StageArgs.cs", "ICustomUI.cs", "MapsDirectlyToDatabaseTableStatelessDefinition.cs", "IObjectUsedByOtherObjectNode.cs", "IInjectKnown.cs", "Themes.cs", "TableView.cs", "TreeView.cs", - "MemoryCatalogueRepository.cs" + "MemoryCatalogueRepository.cs","ExecuteCommandExportCataloguesToHDRGateway.cs" }; public CsProjFileTidy(FileInfo csProjFile) diff --git a/Rdmp.UI.Tests/DesignPatternTests/EvaluateNamespacesAndSolutionFoldersTests.cs b/Rdmp.UI.Tests/DesignPatternTests/EvaluateNamespacesAndSolutionFoldersTests.cs index 0539d93c37..69bb58594f 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/EvaluateNamespacesAndSolutionFoldersTests.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/EvaluateNamespacesAndSolutionFoldersTests.cs @@ -34,8 +34,7 @@ public class EvaluateNamespacesAndSolutionFoldersTests : DatabaseTests "ProjectInstaller.cs", "ProjectInstaller.Designer.cs", "TableView.cs", - "TreeView.cs", - "ExecuteCommandExportCataloguesToHDRGateway.cs" + "TreeView.cs" }; [Test] From c62ee44679f869ccbfbb52d3cf081e0f38166057 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 16 Dec 2025 14:52:17 +0000 Subject: [PATCH 11/13] add docs --- .../Catalogues/ExportCataloguesToHDR.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Documentation/Catalogues/ExportCataloguesToHDR.md diff --git a/Documentation/Catalogues/ExportCataloguesToHDR.md b/Documentation/Catalogues/ExportCataloguesToHDR.md new file mode 100644 index 0000000000..ea2a5ade13 --- /dev/null +++ b/Documentation/Catalogues/ExportCataloguesToHDR.md @@ -0,0 +1,30 @@ +## Export Catalogues to HDR + +The [HDR Gateway](https://healthdatagateway.org/en) is a metadata catalogue for all things health data. +RDMP has the ability to upload all non-internal, non-depricated, non-project specific catalogues to it using the following command +``` +rdmp.exe exportcataloguestohdrgateway https://api.hdruk.cloud/api {teamID} {AppID} {ClientID} {additional config file} + +``` + +Running this command will create new dataets on the HDR Gateway, as well as update and existing datasets with the same name as the catalogue. + +### Command Variables +* teamId - the TeamID that you and the dataset(s) you want to create belong to +* AppID - This integration requires the use of a custom integration, the AppID is provided by this integration +* ClientID- This integration requires the use of a custom integration, the clientID is provided by this integration +* Additional config file - some details are not stored within RDMP, but are typically the same for all datasets that belong to a single owner. This file is used to populate these fields. Use the example below as a starting point + +### Example Config File + { + "accessRights":"https://www.dundee.ac.uk/hic/governance-service", + "accessService":"HIC has implemented a remote-access Trusted Research Environment to protect data confidentiality, satisfy public concerns about data loss and reassure Data Controllers about HIC’s secure management and processing of their data.\r\nData is not released externally to data users for analysis on their own computers but placed on a server at HIC, within a restricted, secure IT environment, where the data user is given secure remote access to carry out their analysis.\r\nFull details are available via the following link: https://www.dundee.ac.uk/hic/safe-haven", + "accessServiceCategory":"TRE/SDE", + "conformsTo":["LOCAL"], + "vocabularyEncodingScheme":["LOCAL"], + "language":["en"], + "format":["CSV","Database"] + "dataUseLimitation":["General research use"], + "resourceCreator":"Please cite us!", + "dataUseRequirements":["Disclosure control"] + } \ No newline at end of file From 2102fa4ac0be3a8228d3e83972125f6565217f66 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 17 Dec 2025 15:20:50 +0000 Subject: [PATCH 12/13] tidy up --- Documentation/Catalogues/ExportCataloguesToHDR.md | 2 +- .../ExecuteCommandExportCataloguesToHDRGateway.cs | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Documentation/Catalogues/ExportCataloguesToHDR.md b/Documentation/Catalogues/ExportCataloguesToHDR.md index ea2a5ade13..2eb0d06690 100644 --- a/Documentation/Catalogues/ExportCataloguesToHDR.md +++ b/Documentation/Catalogues/ExportCataloguesToHDR.md @@ -7,7 +7,7 @@ rdmp.exe exportcataloguestohdrgateway https://api.hdruk.cloud/api {teamID} {AppI ``` -Running this command will create new dataets on the HDR Gateway, as well as update and existing datasets with the same name as the catalogue. +Running this command will create new dataets on the HDR Gateway, as well as update an existing datasets with the same name as the catalogue. ### Command Variables * teamId - the TeamID that you and the dataset(s) you want to create belong to diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs index 95f6ca3071..2d7ac3f366 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportCataloguesToHDRGateway.cs @@ -27,7 +27,7 @@ public class ExecuteCommandExportCataloguesToHDRGateway : BasicCommandExecution, private readonly string _clientID; private readonly HttpClient _client = new(); private readonly string _datasetsEndpoint = "/v2/datasets"; - private InputJSONDetails _inputJsonDetails = new(); + private readonly InputJSONDetails _inputJsonDetails = new(); private class InputJSONDetails { /// @@ -329,7 +329,7 @@ private static string MapPurpose(Catalogue.DatasetPurpose purpose) private class DataTypeObject { public string name; - public List subTypes = new(); + public List subTypes = []; } private static string MapCollectionSource(string source) @@ -617,8 +617,8 @@ public HDRDatasetUpdateInterMetadata(HDRDatasetObject datasetObject) } public HDRDatasetUpdateMetadata metadata { get; set; } - public string schemaModel = "HDRUK"; - public string schemaVersion = "4.0.0"; + public readonly string schemaModel = "HDRUK"; + public readonly string schemaVersion = "4.0.0"; } private class HDRDatasetUpdateBody @@ -646,7 +646,6 @@ public ExecuteCommandExportCataloguesToHDRGateway(IBasicActivateItems activator, _team = team; _appID = appID; _clientID = clientID; - var x = File.ReadAllText(configFile.Name); _inputJsonDetails = JsonConvert.DeserializeObject(File.ReadAllText(configFile.Name)); } From f78acfe17fa522957ee98a7865275684defb1d49 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 17 Dec 2025 15:22:56 +0000 Subject: [PATCH 13/13] any --- Documentation/Catalogues/ExportCataloguesToHDR.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/Catalogues/ExportCataloguesToHDR.md b/Documentation/Catalogues/ExportCataloguesToHDR.md index 2eb0d06690..3699fe60df 100644 --- a/Documentation/Catalogues/ExportCataloguesToHDR.md +++ b/Documentation/Catalogues/ExportCataloguesToHDR.md @@ -7,7 +7,7 @@ rdmp.exe exportcataloguestohdrgateway https://api.hdruk.cloud/api {teamID} {AppI ``` -Running this command will create new dataets on the HDR Gateway, as well as update an existing datasets with the same name as the catalogue. +Running this command will create new dataets on the HDR Gateway, as well as update any existing datasets with the same name as the catalogue. ### Command Variables * teamId - the TeamID that you and the dataset(s) you want to create belong to @@ -27,4 +27,4 @@ Running this command will create new dataets on the HDR Gateway, as well as upda "dataUseLimitation":["General research use"], "resourceCreator":"Please cite us!", "dataUseRequirements":["Disclosure control"] - } \ No newline at end of file + }