[Datasets] Task/rdmp 33 jira asset datasets #2189

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft

JFriel wants to merge 38 commits into task/RDMP-33-dataset-integration-interface from task/RDMP-33-jira-asset-datasets

Directory.Packages.props

-Original file line number
+Diff line change
@@ Expand Up / @@ -10,6 +10,7 @@ @@
         <PackageVersion Include="FluentFTP" Version="53.0.2" />
         <PackageVersion Include="HIC.SynthEHR" Version="2.0.1" />
         <PackageVersion Include="HIC.FAnsiSql" Version="3.2.7" />
+        <PackageVersion Include="JustEat.HttpClientInterception" Version="5.1.1" />
         <PackageVersion Include="LibArchive.Net" Version="0.1.5" />
         <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.10" />
         <PackageVersion Include="Microsoft.Net.Sdk.Compilers.Toolset" Version="9.0.300" />
@@ Expand Down @@

...ommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs

-Original file line number
+Diff line change
@@ Expand Up / @@ -21,7 +21,8 @@ public void TestImportInternalCataloguesOnly() @@
                 var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2");
                 _cata1.SaveToDatabase();
                 _cata2.SaveToDatabase();
+                var ext = new ExtractableDataSet(GetMockActivator().RepositoryLocator.DataExportRepository, _cata2);
+                ext.SaveToDatabase();
                 var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test",1,"test");
                 var provider = new InternalDatasetProvider(GetMockActivator(),configuration,null);
                 var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false,true, false, false,false);
@@ Expand Down Expand Up / @@ -52,7 +53,8 @@ public void TestImportDeprecatedCataloguesOnly() @@
                 var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2");
                 _cata1.SaveToDatabase();
                 _cata2.SaveToDatabase();
+                var ext = new ExtractableDataSet(GetMockActivator().RepositoryLocator.DataExportRepository, _cata2);
+                ext.SaveToDatabase();
                 var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test", 1, "test");
                 var provider = new InternalDatasetProvider(GetMockActivator(), configuration, null);
                 var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false, false, false, true, false);
@@ Expand Down @@

Rdmp.Core.Tests/Datasets/Jira/JiraDatasetProviderTests.cs

-Original file line number
+Diff line change
@@ -0,0 +1,334 @@
+    using Amazon.Runtime.Internal.Endpoints.StandardLibrary;
+    using JustEat.HttpClientInterception;
+    using NUnit.Framework;
+    using Org.BouncyCastle.Asn1.X509;
+    using Rdmp.Core.CommandExecution;
+    using Rdmp.Core.Curation.Data;
+    using Rdmp.Core.Curation.Data.Datasets;
+    using Rdmp.Core.Curation.Data.Datasets.Jira;
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Net.Http;
+    using Tests.Common;
+    using static System.Net.WebRequestMethods;
+    namespace Rdmp.Core.Tests.Datasets.Jira;
+    class JiraDatasetProviderTests : DatabaseTests
+    {
+        private DatasetProviderConfiguration _configuration;
+        private JiraDatasetProvider _provider;
+        private Catalogue _catalogue;
+        private HttpClient _mockHttp;
+        private HttpClientInterceptorOptions options = new HttpClientInterceptorOptions { ThrowOnMissingRegistration = true };
+        [TearDown]
+        public void OneTimeTeardown()
+        {
+            _mockHttp.Dispose();
+        }
+        [OneTimeSetUp]
+        public void OneTimeSetup()
+        {
+            var dataAccessCredentials = new DataAccessCredentials(CatalogueRepository, "name")
+            {
+                Username = "test",
+                Password = "test"
+            };
+            dataAccessCredentials.SaveToDatabase();
+            _configuration = new DatasetProviderConfiguration(CatalogueRepository, "Test Provider", typeof(JiraDatasetProvider).ToString(), "4", dataAccessCredentials.ID, "1234");
+            _configuration.SaveToDatabase();
+            _provider = new JiraDatasetProvider(new ThrowImmediatelyActivator(RepositoryLocator), _configuration, _mockHttp);
+            _catalogue = new Catalogue(CatalogueRepository, "Jira test Catalogue");
+            _catalogue.SaveToDatabase();
+        }
+        private void SetupHttpClient(HttpRequestInterceptionBuilder builder)
+        {
+            builder.RegisterWith(options);
+            _mockHttp = options.CreateHttpClient();
+            _provider = new JiraDatasetProvider(new ThrowImmediatelyActivator(RepositoryLocator), _configuration, _mockHttp);
+        }
+        [Test]
+        public void CreateProviderTest()
+        {
+            Assert.DoesNotThrow(() => _configuration.GetProviderInstance(new ThrowImmediatelyActivator(RepositoryLocator)));
+        }
+        //AddExistingDataset
+        [Test]
+        public void AddExistingDatasetTest_Success()
+        {
+            var url = "999";
+            var builder = new HttpRequestInterceptionBuilder()
+                .Requests()
+                .ForHttps()
+                .ForHost("api.atlassian.com/jsm/assets/workspace")
+                .ForPath($"/1234/v1/object/{url}")
+                .Responds()
+                .WithStatus(200).WithJsonContent(SucessResponseString);
+            SetupHttpClient(builder);
+            string name = "AddExistingDatasetTest_Success";
+            Assert.DoesNotThrow(() => _provider.AddExistingDataset(name, url));
+            var dataset = CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Datasets.Dataset>().FirstOrDefault(d => d.Name == name);
+            Assert.That(dataset, Is.Not.Null);
+            dataset.DeleteInDatabase();
+        }
+        [Test]
+        public void AddExistingDatasetTest_BadRemote()
+        {
+            var url = "999";
+            var builder = new HttpRequestInterceptionBuilder()
+                .Requests()
+                .ForHttps()
+                .ForHost("api.atlassian.com/jsm/assets/workspace")
+                .ForPath($"/1234/v1/object/{url}")
+                .Responds()
+                .WithStatus(400);
+            SetupHttpClient(builder);
+            string name = "AddExistingDatasetTest_BadRemote";
+            Assert.Throws<Exception>(() => _provider.AddExistingDataset(name, url));
+            var dataset = CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Datasets.Dataset>().FirstOrDefault(d => d.Name == name);
+            Assert.That(dataset, Is.Null);
+        }
+        //AddExistingDatasetWithReturn
+        [Test]
+        public void AddExistingDatasetWithReturnTest_Success()
+        {
+            var url = "999";
+            var builder = new HttpRequestInterceptionBuilder()
+                .Requests()
+                .ForHttps()
+                .ForHost("api.atlassian.com/jsm/assets/workspace")
+                .ForPath($"/1234/v1/object/{url}")
+                .Responds()
+                .WithStatus(200).WithJsonContent(SucessResponseString);
+            SetupHttpClient(builder);
+            string name = "AddExistingDatasetWithReturnTest_Success";
+            Assert.DoesNotThrow(() => _provider.AddExistingDatasetWithReturn(name, url));
+            var dataset = CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Datasets.Dataset>().FirstOrDefault(d => d.Name == name);
+            Assert.That(dataset, Is.Not.Null);
+            dataset.DeleteInDatabase();
+        }
+        [Test]
+        public void AddExistingDatasetWithReturnTest_BadRemote()
+        {
+            var url = "999";
+            var builder = new HttpRequestInterceptionBuilder()
+                .Requests()
+                .ForHttps()
+                .ForHost("api.atlassian.com/jsm/assets/workspace")
+                .ForPath($"/1234/v1/object/{url}")
+                .Responds()
+                .WithStatus(400);
+            SetupHttpClient(builder);
+            string name = "AddExistingDatasetWithReturnTest_BadRemote";
+            Assert.Throws<Exception>(() => _provider.AddExistingDatasetWithReturn(name, url));
+            var dataset = CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Datasets.Dataset>().FirstOrDefault(d => d.Name == name);
+            Assert.That(dataset, Is.Null);
+        }
+        [Test]
+        public void CreateDataset_BadObjectType()
+        {
+            var builder = new HttpRequestInterceptionBuilder()
+            .Requests()
+         .ForHttps()
+            .ForHost("api.atlassian.com/jsm/assets/workspace")
+            .ForPath($"/1234/v1/objectschema/4/objecttypes")
+            .Responds()
+            .WithStatus(400);
+            SetupHttpClient(builder);
+            builder = new HttpRequestInterceptionBuilder()
+                .Requests()
+                .ForPost()
+            .ForHttps()
+               .ForHost("api.atlassian.com/jsm/assets/workspace")
+               .ForPath($"/1234/v1/object/create")
+               .Responds()
+               .WithStatus(201).WithJsonContent(SucessCreateString);
+            SetupHttpClient(builder);
+            builder = new HttpRequestInterceptionBuilder()
+                .Requests().ForHttps()
+                .ForHost("api.atlassian.com/jsm/assets/workspace")
+                .ForPath($"/1234/v1/objecttype/1/attributes")
+                .Responds()
+                .WithStatus(200).WithJsonContent(SuccessObjectTypeAttributes);
+            SetupHttpClient(builder);
+            builder = new HttpRequestInterceptionBuilder()
+               .Requests()
+               .ForHttps()
+               .ForHost("api.atlassian.com/jsm/assets/workspace")
+               .ForPath($"/1234/v1/object/999")
+               .Responds()
+               .WithStatus(200).WithJsonContent(SucessResponseString);
+            SetupHttpClient(builder);
+            var catalogue = new Catalogue(CatalogueRepository, "CreateDataset_Success");
+            catalogue.SaveToDatabase();
+            var e = Assert.Throws<Exception>(() => _provider.Create(catalogue));
+            Assert.That(e.Message, Is.EqualTo("BadRequest: Unable to fetch Object Types"));
+        }
+        [Test]
+        public void CreateDataset_UnableToCreate()
+        {
+            var builder = new HttpRequestInterceptionBuilder()
+            .Requests()
+         .ForHttps()
+            .ForHost("api.atlassian.com/jsm/assets/workspace")
+            .ForPath($"/1234/v1/objectschema/4/objecttypes")
+            .Responds()
+            .WithStatus(200).WithJsonContent(SucessObjectTypesString);
+            SetupHttpClient(builder);
+            builder = new HttpRequestInterceptionBuilder()
+                .Requests()
+                .ForPost()
+            .ForHttps()
+               .ForHost("api.atlassian.com/jsm/assets/workspace")
+               .ForPath($"/1234/v1/object/create")
+               .Responds()
+               .WithStatus(400);
+            SetupHttpClient(builder);
+            builder = new HttpRequestInterceptionBuilder()
+                .Requests().ForHttps()
+                .ForHost("api.atlassian.com/jsm/assets/workspace")
+                .ForPath($"/1234/v1/objecttype/1/attributes")
+                .Responds()
+                .WithStatus(200).WithJsonContent(SuccessObjectTypeAttributes);
+            SetupHttpClient(builder);
+            builder = new HttpRequestInterceptionBuilder()
+               .Requests()
+               .ForHttps()
+               .ForHost("api.atlassian.com/jsm/assets/workspace")
+               .ForPath($"/1234/v1/object/999")
+               .Responds()
+               .WithStatus(200).WithJsonContent(SucessResponseString);
+            SetupHttpClient(builder);
+            var catalogue = new Catalogue(CatalogueRepository, "CreateDataset_Success");
+            catalogue.SaveToDatabase();
+            var e = Assert.Throws<Exception>(() => _provider.Create(catalogue));
+            Assert.That(e.Message, Is.EqualTo("BadRequest: Unable to create Dataset"));
+        }
+        [Test]
+        public void CreateDataset_NoMatchingSchema()
+        {
+            var builder = new HttpRequestInterceptionBuilder()
+                   .Requests()
+                .ForHttps()
+                   .ForHost("api.atlassian.com/jsm/assets/workspace")
+                   .ForPath($"/1234/v1/objectschema/4/objecttypes")
+                   .Responds()
+                   .WithStatus(200).WithJsonContent(SucessObjectTypesString);
+            SetupHttpClient(builder);
+            builder = new HttpRequestInterceptionBuilder()
+                .Requests()
+                .ForPost()
+            .ForHttps()
+               .ForHost("api.atlassian.com/jsm/assets/workspace")
+               .ForPath($"/1234/v1/object/create")
+               .Responds()
+               .WithStatus(201).WithJsonContent(SucessCreateString);
+            SetupHttpClient(builder);
+            builder = new HttpRequestInterceptionBuilder()
+                .Requests().ForHttps()
+                .ForHost("api.atlassian.com/jsm/assets/workspace")
+                .ForPath($"/1234/v1/objecttype/1/attributes")
+                .Responds()
+                .WithStatus(400);
+            SetupHttpClient(builder);
+            builder = new HttpRequestInterceptionBuilder()
+               .Requests()
+               .ForHttps()
+               .ForHost("api.atlassian.com/jsm/assets/workspace")
+               .ForPath($"/1234/v1/object/999")
+               .Responds()
+               .WithStatus(200).WithJsonContent(SucessResponseString);
+            SetupHttpClient(builder);
+            var catalogue = new Catalogue(CatalogueRepository, "CreateDataset_Success");
+            catalogue.SaveToDatabase();
+            var e = Assert.Throws<Exception>(() => _provider.Create(catalogue));
+            Assert.That(e.Message, Is.EqualTo("BadRequest: Unable to fetch Object Attributes for Schema 4"));
+        }
+        //FetchDatasetByID
+        [Test]
+        public void FetchDatasetByIDTest_Success()
+        {
+            var url = "999";
+            var builder = new HttpRequestInterceptionBuilder()
+                .Requests()
+                .ForHttps()
+                .ForHost("api.atlassian.com/jsm/assets/workspace")
+                .ForPath($"/1234/v1/object/{url}")
+                .Responds()
+                .WithStatus(200).WithJsonContent(SucessResponseString);
+            SetupHttpClient(builder);
+            Assert.DoesNotThrow(() => _provider.FetchDatasetByID(int.Parse(url)));
+        }
+        [Test]
+        public void FetchDatasetByIDTest_BadRemote()
+        {
+            var url = "999";
+            var builder = new HttpRequestInterceptionBuilder()
+                .Requests()
+                .ForHttps()
+                .ForHost("api.atlassian.com/jsm/assets/workspace")
+                .ForPath($"/1234/v1/object/{url}")
+                .Responds()
+                .WithStatus(400);
+            SetupHttpClient(builder);
+            Assert.Throws<Exception>(() => _provider.FetchDatasetByID(int.Parse(url)));
+        }
+        private object SucessResponseString = new
+        {
+            id = "22",
+            name = "AddExistingDatasetWithReturnTest_Success",
+            type = "Dataset",
+            url = "22",
+            source = "Jira",
+            _links = new
+            {
+                self = "https://api.atlassian.com/jsm/assets/workspace/1234/v1/object/22",
+            }
+        };
+        private object SucessCreateString = new
+        {
+        };
+        private List<object> SucessObjectTypesString = [new { name = "Dataset", id = 1, objectSchemaId = 4 }];
+        private List<object> SuccessObjectTypeAttributes = [
+            new {name="Name",id=2 }
+            ];
+        private object SucessAQLResult = new
+        {
+            values = new List<object>()
+        };
+    }

Rdmp.Core.Tests/Rdmp.Core.Tests.csproj

-Original file line number
+Diff line change
@@ Expand Up / @@ -68,6 +68,7 @@ @@
                 <PrivateAssets>all</PrivateAssets>
                 <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             </PackageReference>
+            <PackageReference Include="JustEat.HttpClientInterception" />
             <PackageReference Include="Microsoft.NET.Test.Sdk" />
             <PackageReference Include="Minio" />
             <PackageReference Include="NSubstitute" />
@@ Expand Down @@

...ution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs

-Original file line number
+Diff line change
@@ Expand Up / @@ -60,6 +60,7 @@ public override void Execute() @@
                 foreach (var catalogue in catalogues)
                 {
                     var dataset = _provider.Create(catalogue);
                     var ds = _provider.AddExistingDatasetWithReturn(null, dataset.GetID());
                     var cmd = new ExecuteCommandLinkCatalogueToDataset(_activator, catalogue, ds,_autoUpdate);
                     cmd.Execute();
@@ Expand Down @@

Rdmp.Core/Curation/Data/Datasets/IDataset.cs

-Original file line number
+Diff line change
@@ Expand Up / @@ -35,11 +35,13 @@ public interface IDataset: IMapsDirectlyToDatabaseTable @@
         /// </summary>
         string Type { get; }
         /// <summary>
         /// The URL to access the dataset
         /// </summary>
         string Url { get; }
         public abstract string GetID();
         public abstract string GetRemoteID();
     }

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[Datasets] Task/rdmp 33 jira asset datasets #2189

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

[Datasets] Task/rdmp 33 jira asset datasets #2189

Are you sure you want to change the base?

Uh oh!

[Datasets] Task/rdmp 33 jira asset datasets #2189

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!