diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDataset.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDataset.cs new file mode 100644 index 0000000000..d324bbd8cc --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDataset.cs @@ -0,0 +1,106 @@ +using Microsoft.VisualBasic; +using Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure +{ +#nullable enable + /// + /// Used for mapping Pure datasets from the API into a C# object. + /// + public class PureDataset : PluginDataset + { + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? PureId { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UUID { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? CreatedDate { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + + public string? CreatedBy { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + + public string? ModifiedDate { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + + public string? ModifiedBy { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? PortalURL { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Version { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ENGBWrapper? Title { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Descriptions { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public PureSystem? ManagingOrganization { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public new URITerm? Type { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public PureSystem? Publisher { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Geolocation? Geolocation { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Persons { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Organizations { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public PureDate? PublicationAvailableDate { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? OpenAireCompliant { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Visibility? Visibility { get; set; } + + //TODO custom defined fields + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Workflow? Workflow { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? SystemName { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + + public TemporalCoveragePeriod? TemporalCoveragePeriod { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List Links { get; set; } + + + public override string GetRemoteID() + { + return Url.Split("/").Last(); + } + +#nullable disable + + public PureDataset() { } + public PureDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name) { } + public PureDataset(ICatalogueRepository repository, DbDataReader r) : base(repository, r) { } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/ENGBWrapper.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/ENGBWrapper.cs new file mode 100644 index 0000000000..59fd0d4c97 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/ENGBWrapper.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + + /// + /// Internal PURE system class + /// + public class ENGBWrapper + { + public ENGBWrapper(string? text) { En_GB = text; } + public string? En_GB { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/Geolocation.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/Geolocation.cs new file mode 100644 index 0000000000..6f81ea1681 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/Geolocation.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + /// + /// Internal PURE system class + /// + public class Geolocation + { + public ENGBWrapper? GeographicalCoverage { get; set; } + public string? Point { get; set; } + + public string? Polygon { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureDate.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureDate.cs new file mode 100644 index 0000000000..88e8feaeef --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureDate.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + /// + /// Internal PURE system class + /// + public class PureDate + { + public PureDate(DateTime dateTime) + { + Year = dateTime.Year; + Month = dateTime.Month; + Day = dateTime.Day; + } + + public PureDate() { } + + + public DateTime ToDateTime() + { + return new DateTime(Year, Month ?? 1, Day ?? 1, 0, 0, 0); + } + + public bool IsBefore(PureDate date) + { + if (Year < date.Year) return true; + if (Year == date.Year) + { + if (Month < date.Month) return true; + if (Month == date.Month) + { + return Day < date.Day; + } + } + + return false; + } + + public PureDate(int year, int? month = null, int? day = null) + { + Year = year; + if (month != null) Month = month; + if (day != null) Day = day; + } + public int Year { get; set; } + public int? Month { get; set; } + public int? Day { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureDescription.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureDescription.cs new file mode 100644 index 0000000000..3bd91878be --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureDescription.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + + /// + /// Internal PURE system class + /// + public class PureDescription + { + public int? PureId { get; set; } + public ENGBWrapper? Value { get; set; } + + public URITerm Term { get => new URITerm("/dk/atira/pure/dataset/descriptions/datasetdescription", new ENGBWrapper("Description")); } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureLink.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureLink.cs new file mode 100644 index 0000000000..e58f15bd35 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureLink.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + /// + /// Link to a PURE object + /// + public class PureLink + { + + public PureLink(int pureID, string url, string? alias, ENGBWrapper description, URITerm linkType) + { + PureID = pureID; + Url = url; + Alias = alias; + Description = description; + LinkType = linkType; + } + public int PureID { get; set; } + public string Url { get; set; } + + public string? Alias { get; set; } + public ENGBWrapper Description { get; set; } + public URITerm LinkType { get; set; } + + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureName.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureName.cs new file mode 100644 index 0000000000..009e715de0 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureName.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + /// + /// Internal PURE system class + /// + public class PureName() + { + public string? FirstName { get; set; } + public string? LastName { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PurePerson.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PurePerson.cs new file mode 100644 index 0000000000..c953a95dd6 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PurePerson.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +# nullable enable + /// + /// Internal PURE system class + /// + public class PurePerson + { + public string? TypeDiscriminator { get; set; } + public int? PureId { get; set; } + + public PureName? Name { get; set; } + public URITerm? Role { get; set; } + + public List? Organizations { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureSystem.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureSystem.cs new file mode 100644 index 0000000000..0e31a0aee1 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/PureSystem.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + /// + /// Internal PURE system class + /// + public class PureSystem + { + public PureSystem(string? uuid, string? systemName) + { + UUID = uuid; + SystemName = systemName; + } + public string? SystemName { get; set; } + public string? UUID { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/TemporalCoveragePeriod.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/TemporalCoveragePeriod.cs new file mode 100644 index 0000000000..6a30c81c74 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/TemporalCoveragePeriod.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + /// + /// Internal PURE system class + /// + public class TemporalCoveragePeriod + { + public PureDate? StartDate { get; set; } + public PureDate? EndDate { get; set; } + } +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/URITerm.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/URITerm.cs new file mode 100644 index 0000000000..1278c26e92 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/URITerm.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + /// + /// Internal PURE system class + /// + public class URITerm + { + public URITerm(string? uri, ENGBWrapper enGBWrapper) + { + URI = uri; + Term = enGBWrapper; + } + public string? URI { get; set; } + public ENGBWrapper Term { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/Visibility.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/Visibility.cs new file mode 100644 index 0000000000..4aaa56d636 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/Visibility.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + /// + /// Internal PURE system class + /// + public class Visibility + { + public Visibility(string? key, ENGBWrapper description) + { + Key = key; + Description = description; + } + public string? Key { get; set; } + public ENGBWrapper Description { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/Workflow.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/Workflow.cs new file mode 100644 index 0000000000..e7b6c84600 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetItem/Workflow.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem +{ +#nullable enable + /// + /// Internal PURE system class + /// + public class Workflow + { + public string? Step { get; set; } + public ENGBWrapper? Description { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetProvider.cs new file mode 100644 index 0000000000..af474361fb --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Pure/PureDatasetProvider.cs @@ -0,0 +1,181 @@ +using Amazon.Runtime.Internal.Endpoints.StandardLibrary; +using Newtonsoft.Json; +using NPOI.OpenXmlFormats.Dml.Diagram; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data.Datasets.Pure.PureDatasetItem; +using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Pure +{ + class PureDatasetProvider : PluginDatasetProvider + { + private HttpClient _client; + private readonly string _url; + public PureDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration, HttpClient client = null) : base(activator, configuration) + { + _client = new HttpClient(); + var credentials = Repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); + var apiKey = credentials.GetDecryptedPassword(); + _client.DefaultRequestHeaders.Add("api-key", apiKey); + _url = Configuration.Url.TrimEnd('/'); + } + public override void AddExistingDataset(string name, string url) + { + AddExistingDatasetWithReturn(name, url); + } + + public override Dataset AddExistingDatasetWithReturn(string name, string url) + { + var uri = $"{_url}/data-sets/{UrltoUUID(url)}"; + var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + PureDataset pd = JsonConvert.DeserializeObject(detailsString); + var datasetName = string.IsNullOrWhiteSpace(name) ? pd.Title.En_GB : name; + var dataset = new Curation.Data.Datasets.Dataset(Repository, datasetName) + { + Url = url, + Type = this.ToString(), + Provider_ID = Configuration.ID, + DigitalObjectIdentifier = pd.DigitalObjectIdentifier, + Folder = $"\\Pure\\{Configuration.Name}", + }; + dataset.SaveToDatabase(); + Activator.Publish(dataset); + return dataset; + } + else + { + throw new Exception("Cannot access dataset at provided url"); + } + } + + public override Dataset FetchDatasetByID(int id) + { + var uri = $"{_url}/data-sets/{UrltoUUID(id.ToString())}"; + var response = Task.Run(async () => await _client.GetAsync(uri)).Result; + if (response.StatusCode != HttpStatusCode.OK) + { + throw new Exception($"Dataset with ID '{id}' does cannot be found"); + } + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + PureDataset pd = JsonConvert.DeserializeObject(detailsString); + return pd; + } + + public override void Update(string uuid, PluginDataset datasetUpdates) + { + var serializeOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }; + + var options = new JsonWriterOptions + { + Indented = true + }; + + using var stream = new MemoryStream(); + System.Text.Json.JsonSerializer.Serialize(stream, (PureDataset)datasetUpdates, serializeOptions); + var jsonString = Encoding.UTF8.GetString(stream.ToArray()); + var uri = $"{Configuration.Url}/data-sets/{uuid}"; + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + + var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; + if (response.StatusCode != HttpStatusCode.OK) + { + throw new Exception("Error"); + } + } + + public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) + { + var existingDataset = (PureDataset)FetchDatasetByID(int.Parse(dataset.Url.Split("/").Last())); + var datasetUpdate = new PureDataset(); + datasetUpdate.Title = new ENGBWrapper(catalogue.Name); + var datasetDescriptionTerm = "/dk/atira/pure/dataset/descriptions/datasetdescription"; + var description = existingDataset.Descriptions.Where(d => d.Term.URI == datasetDescriptionTerm).FirstOrDefault(); + if (description is null) + { + //error + Console.WriteLine("No known description!"); + } + else + { + description.Value = new ENGBWrapper(catalogue.Description); + datasetUpdate.Descriptions = new List { description }; + } + if (catalogue.StartDate != null) + { + datasetUpdate.TemporalCoveragePeriod.StartDate = new PureDate((DateTime)catalogue.StartDate); + } + if (catalogue.EndDate != null) + { + datasetUpdate.TemporalCoveragePeriod.EndDate = new PureDate((DateTime)catalogue.EndDate); + } + if (catalogue.Geographical_coverage != null) + { + datasetUpdate.Geolocation = new Geolocation(); + datasetUpdate.Geolocation.GeographicalCoverage = new ENGBWrapper(catalogue.Geographical_coverage); + if (existingDataset.Geolocation != null) + { + datasetUpdate.Geolocation.Point = existingDataset.Geolocation.Point; + datasetUpdate.Geolocation.Polygon = existingDataset.Geolocation.Polygon; + } + } + Update(existingDataset.UUID, datasetUpdate); + } + + public override Dataset Create(Catalogue catalogue) + { + var uri = $"{_url}/data-sets/"; + var newDataset = new PureDataset(); + newDataset.Title = new ENGBWrapper(catalogue.Name); + + var serializeOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }; + + var options = new JsonWriterOptions + { + Indented = true + }; + + using var stream = new MemoryStream(); + System.Text.Json.JsonSerializer.Serialize(stream, newDataset, serializeOptions); + var jsonString = Encoding.UTF8.GetString(stream.ToArray()); + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + + var response = Task.Run(async () => await _client.PutAsync(uri, httpContent)).Result; + if (response.StatusCode != HttpStatusCode.OK) + { + throw new Exception($"Error: {response.StatusCode}"); + } + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + PureDataset pd = JsonConvert.DeserializeObject(detailsString); + UpdateUsingCatalogue(pd, catalogue); + return FetchDatasetByID(pd.PureId.Value); + } + + public override string GetRemoteURL(Dataset dataset) + { + var existingDataset = (PureDataset)FetchDatasetByID(int.Parse(dataset.Url.Split("/").Last())); + return existingDataset.PortalURL; + } + private static string UrltoUUID(string url) => url.Split("/").Last(); + } +} diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index fe1613e925..3600fbb181 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -29,6 +29,7 @@ using Rdmp.Core.Curation.Data.Dashboarding; using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data.Datasets.Pure; using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Pipelines; @@ -622,6 +623,10 @@ public static T WhenIHaveA(MemoryDataExportRepository repository) where T : D { return (T)(object)new DatasetProviderConfiguration(repository.CatalogueRepository, "","","",WhenIHaveA(repository).ID,""); } + if (typeof(T) == typeof(PureDataset)) + { + return (T)(object)new PureDataset(repository.CatalogueRepository, "Pure Dataset"); + } throw new TestCaseNotWrittenYetException(typeof(T)); }