[Datasets] Task/rdmp 33 hdr datasets #2188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft

JFriel wants to merge 29 commits into task/RDMP-33-dataset-integration-interface from task/RDMP-33-hdr-datasets

Directory.Packages.props

-Original file line number
+Diff line change
@@ Expand Up / @@ -10,6 +10,7 @@ @@
         <PackageVersion Include="FluentFTP" Version="53.0.1" />
         <PackageVersion Include="HIC.SynthEHR" Version="2.0.1" />
         <PackageVersion Include="HIC.FAnsiSql" Version="3.2.7" />
+        <PackageVersion Include="JustEat.HttpClientInterception" Version="5.1.1" />
         <PackageVersion Include="LibArchive.Net" Version="0.1.5" />
         <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.7" />
         <PackageVersion Include="Microsoft.Net.Sdk.Compilers.Toolset" Version="9.0.300" />
@@ Expand Down @@

Rdmp.Core.Tests/Datasets/HDR/HDRDatasetProviderTests.cs

-Original file line number
+Diff line change
@@ -0,0 +1,199 @@
+    using NUnit.Framework;
+    using Rdmp.Core.CommandExecution;
+    using Rdmp.Core.Curation.Data.Datasets;
+    using Rdmp.Core.Curation.Data;
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Net.Http;
+    using System.Text;
+    using System.Threading.Tasks;
+    using Tests.Common;
+    using Rdmp.Core.Curation.Data.Datasets.HDR;
+    using JustEat.HttpClientInterception;
+    namespace Rdmp.Core.Tests.Datasets.HDR
+    {
+        class HDRDatasetProviderTests : DatabaseTests
+        {
+            private DatasetProviderConfiguration _configuration;
+            private HDRDatasetProvider _provider;
+            private Catalogue _catalogue;
+            private HttpClient _mockHttp;
+            private HttpClientInterceptorOptions options = new HttpClientInterceptorOptions { ThrowOnMissingRegistration = true };
+            [TearDown]
+            public void OneTimeTeardown()
+            {
+                _mockHttp.Dispose();
+            }
+            [OneTimeSetUp]
+            public void OneTimeSetup()
+            {
+                var dataAccessCredentials = new DataAccessCredentials(CatalogueRepository, "name")
+                {
+                    Username = "test",
+                    Password = "test"
+                };
+                dataAccessCredentials.SaveToDatabase();
+                _configuration = new DatasetProviderConfiguration(CatalogueRepository, "Test Provider", typeof(HDRDatasetProvider).ToString(), "https://doesNotExist.preprod.hdruk.cloud/api", dataAccessCredentials.ID, "1234");
+                _configuration.SaveToDatabase();
+                _provider = new HDRDatasetProvider(new ThrowImmediatelyActivator(RepositoryLocator), _configuration, _mockHttp);
+                _catalogue = new Catalogue(CatalogueRepository, "HDR test Catalogue");
+                _catalogue.SaveToDatabase();
+            }
+            private void SetupHttpClient(HttpRequestInterceptionBuilder builder)
+            {
+                builder.RegisterWith(options);
+                _mockHttp = options.CreateHttpClient();
+                _provider = new HDRDatasetProvider(new ThrowImmediatelyActivator(RepositoryLocator), _configuration, _mockHttp);
+            }
+            [Test]
+            public void CreateProviderTest()
+            {
+                Assert.DoesNotThrow(() => _configuration.GetProviderInstance(new ThrowImmediatelyActivator(RepositoryLocator)));
+            }
+            [Test]
+            public void AddExistingDatasetTest_Success()
+            {
+                var url = "999";
+                var builder = new HttpRequestInterceptionBuilder()
+                    .Requests()
+                    .ForHttps()
+                    .ForHost("doesnotexist.preprod.hdruk.cloud")
+                    .ForPath("api/v1/datasets/999")
+                    .ForQuery("schema_model=HDRUK&schema_version=3.0.0")
+                    .Responds()
+                    .WithStatus(200).WithJsonContent(SucessResponseString);
+                SetupHttpClient(builder);
+                string name = "AddExistingDatasetTest_Success";
+                Assert.DoesNotThrow(() => _provider.AddExistingDataset(name, url));
+                var dataset = CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Datasets.Dataset>().FirstOrDefault(d => d.Name == name);
+                Assert.That(dataset, Is.Not.Null);
+                dataset.DeleteInDatabase();
+            }
+            [Test]
+            public void AddExistingDatasetTest_BadRemote()
+            {
+                var url = "999";
+                var builder = new HttpRequestInterceptionBuilder()
+                    .Requests()
+                    .ForHttps()
+                    .ForHost("doesnotexist.preprod.hdruk.cloud")
+                    .ForPath("api/v1/datasets/999")
+                    .ForQuery("schema_model=HDRUK&schema_version=3.0.0")
+                    .Responds()
+                    .WithStatus(400);
+                SetupHttpClient(builder);
+                string name = "HDRAddExistingDatasetTest_BadRemote";
+                Assert.Throws<Exception>(() => _provider.AddExistingDataset(name, url));
+                var dataset = CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Datasets.Dataset>().FirstOrDefault(d => d.Name == name);
+                Assert.That(dataset, Is.Null);
+            }
+            [Test]
+            public void AddExistingDatasetWithReturnTest_Success()
+            {
+                var url = "999";
+                var builder = new HttpRequestInterceptionBuilder()
+                    .Requests()
+                    .ForHttps()
+                    .ForHost("doesnotexist.preprod.hdruk.cloud")
+                    .ForPath("api/v1/datasets/999")
+                    .ForQuery("schema_model=HDRUK&schema_version=3.0.0")
+                    .Responds()
+                    .WithStatus(200).WithJsonContent(SucessResponseString);
+                SetupHttpClient(builder);
+                string name = "AddExistingDatasetTest_Success";
+                Assert.DoesNotThrow(() => _provider.AddExistingDatasetWithReturn(name, url));
+                var dataset = CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Datasets.Dataset>().FirstOrDefault(d => d.Name == name);
+                Assert.That(dataset, Is.Not.Null);
+                dataset.DeleteInDatabase();
+            }
+            [Test]
+            public void AddExistingDatasetWithReturnTest_BadRemote()
+            {
+                var url = "999";
+                var builder = new HttpRequestInterceptionBuilder()
+                    .Requests()
+                    .ForHttps()
+                    .ForHost("doesnotexist.preprod.hdruk.cloud")
+                    .ForPath("api/v1/datasets/999")
+                    .ForQuery("schema_model=HDRUK&schema_version=3.0.0")
+                    .Responds()
+                    .WithStatus(400);
+                SetupHttpClient(builder);
+                string name = "HDRAddExistingDatasetTest_BadRemote";
+                Assert.Throws<Exception>(() => _provider.AddExistingDatasetWithReturn(name, url));
+                var dataset = CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Datasets.Dataset>().FirstOrDefault(d => d.Name == name);
+                Assert.That(dataset, Is.Null);
+            }
+            //FetchDatasetByID
+            [Test]
+            public void FetchDatasetByIDTest_Success()
+            {
+                var url = "999";
+                var builder = new HttpRequestInterceptionBuilder()
+                    .Requests()
+                    .ForHttps()
+                   .ForHost("doesnotexist.preprod.hdruk.cloud")
+                    .ForPath("api/v1/datasets/999")
+                    .ForQuery("schema_model=HDRUK&schema_version=3.0.0")
+                    .Responds()
+                    .WithStatus(200).WithJsonContent(SucessResponseString);
+                SetupHttpClient(builder);
+                Assert.DoesNotThrow(() => _provider.FetchDatasetByID(int.Parse(url)));
+            }
+            [Test]
+            public void FetchDatasetByIDTest_BadRemote()
+            {
+                var url = "999";
+                var builder = new HttpRequestInterceptionBuilder()
+                    .Requests()
+                    .ForHttps()
+                  .ForHost("doesnotexist.preprod.hdruk.cloud")
+                    .ForPath("api/v1/datasets/999")
+                    .ForQuery("schema_model=HDRUK&schema_version=3.0.0")
+                    .Responds()
+                    .WithStatus(400);
+                SetupHttpClient(builder);
+                Assert.Throws<Exception>(() => _provider.FetchDatasetByID(int.Parse(url)));
+            }
+            private object SucessResponseString = new
+            {
+                id = "22",
+                name = "AddExistingDatasetWithReturnTest_Success",
+                type = "Dataset",
+                url = "22",
+                source = "Jira",
+                _links = new
+                {
+                    self = "https://api.atlassian.com/jsm/assets/workspace/1234/v1/object/22",
+                }
+            };
+            private object SucessCreateString = new
+            {
+            };
+            private List<object> SucessObjectTypesString = [new { name = "Dataset", id = 1, objectSchemaId = 4 }];
+            private List<object> SuccessObjectTypeAttributes = [
+                new {name="Name",id=2 }
+                ];
+            private object SucessAQLResult = new
+            {
+                values = new List<object>()
+            };
+        }
+    }

Rdmp.Core.Tests/Rdmp.Core.Tests.csproj

-Original file line number
+Diff line change
@@ Expand Up / @@ -68,6 +68,7 @@ @@
                 <PrivateAssets>all</PrivateAssets>
                 <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             </PackageReference>
+            <PackageReference Include="JustEat.HttpClientInterception" />
             <PackageReference Include="Microsoft.NET.Test.Sdk" />
             <PackageReference Include="Minio" />
             <PackageReference Include="NSubstitute" />
@@ Expand Down @@

...ution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -33,24 +33,25 @@ public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicA
  
            public List<Catalogue> GetCatalogues()

            {

                var cataloguesToReturn = new List<Catalogue>();

                var catalogues = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects<Catalogue>().ToList();

                if (!_includeInternal)

                if (_includeInternal)

                {

                    catalogues = catalogues.Where(c => !c.IsInternalDataset).ToList();

                    cataloguesToReturn.AddRange(catalogues.Where(c => c.IsInternalDataset).ToList());

                }

                if (!_includeProjectSpecific)

                if (_includeProjectSpecific)

                {

                    catalogues = catalogues.Where(c => !c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)).ToList();

                    cataloguesToReturn.AddRange(catalogues.Where(c => c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)).ToList());

                }

                if (!_includeDeprecated)

                if (_includeDeprecated)

                {

                    catalogues = catalogues.Where(c => !c.IsDeprecated).ToList();

                    cataloguesToReturn.AddRange(catalogues.Where(c => c.IsDeprecated).ToList());

                }

                if (!_includeExtractable)

                if (_includeExtractable)

                {

                    catalogues = catalogues.Where(c => !c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList();

                    cataloguesToReturn.AddRange(catalogues.Where(c => c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList());

                }

                return catalogues;

                return cataloguesToReturn.Distinct().ToList();

            }

Rdmp.Core/Curation/Data/Datasets/HDR/HDRDataset.cs

-Original file line number
+Diff line change
@@ -0,0 +1,40 @@
+    using Rdmp.Core.Repositories;
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Text;
+    using System.Threading.Tasks;
+    namespace Rdmp.Core.Curation.Data.Datasets.HDR
+    {
+        /// <summary>
+        /// Mapping of HDR Dataset object to C#
+        /// </summary>
+        public class HDRDataset : PluginDataset
+        {
+            public HDRDataset() : base() { }
+            public HDRDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name)
+            {
+            }
+            public HDRDatasetItems.Data data { get; set; }
+            public override string GetID()
+            {
+                return data.id.ToString();
+            }
+            public override string GetRemoteID()
+            {
+                return Url.Split('?')[0].Split('/').Last();
+            }
+            public string GetDOI()
+            {
+                var version = data?.versions?.FirstOrDefault();
+                if (version != null) {
+                    return version.metadata?.metadata?.summary?.doiName?.ToString();
+                }
+                return null;
+            }
+        }
+    }

Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Access.cs

-Original file line number
+Diff line change
@@ -0,0 +1,23 @@
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Text;
+    using System.Threading.Tasks;
+    namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems
+    {
+        /// <summary>
+        /// Mapping from HDR API
+        /// </summary>
+        public class Access
+        {
+            public string deliveryLeadTime { get; set; }
+            public List<string> jurisdiction { get; set; }
+            public string dataController { get; set; }
+            public string dataProcessor { get; set; }
+            public string accessRights { get; set; }
+            public string accessService { get; set; }
+            public string accessRequestCost { get; set; }
+            public object accessServiceCategory { get; set; }
+        }
+    }

Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Accessibility.cs

-Original file line number
+Diff line change
@@ -0,0 +1,18 @@
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Text;
+    using System.Threading.Tasks;
+    namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems
+    {
+        /// <summary>
+        /// Mapping from HDR API
+        /// </summary>
+        public class Accessibility
+        {
+            public Access access { get; set; }
+            public Usage usage { get; set; }
+            public FormatAndStandards formatAndStandards { get; set; }
+        }
+    }

Rdmp.Core/Curation/Data/Datasets/HDR/HDRDatasetItems/Coverage.cs

-Original file line number
+Diff line change
@@ -0,0 +1,22 @@
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Text;
+    using System.Threading.Tasks;
+    namespace Rdmp.Core.Curation.Data.Datasets.HDR.HDRDatasetItems
+    {
+        /// <summary>
+        /// Mapping from HDR API
+        /// </summary>
+        public class Coverage
+        {
+            public object pathway { get; set; }
+            public string spatial { get; set; }
+            public object followUp { get; set; }
+            public object datasetCompleteness { get; set; }
+            public List<string> materialType { get; set; }
+            public int typicalAgeRangeMin { get; set; }
+            public int typicalAgeRangeMax { get; set; }
+        }
+    }

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[Datasets] Task/rdmp 33 hdr datasets #2188

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

[Datasets] Task/rdmp 33 hdr datasets #2188

Are you sure you want to change the base?

Uh oh!

[Datasets] Task/rdmp 33 hdr datasets #2188

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!