diff --git a/Directory.Packages.props b/Directory.Packages.props index 311d122b1d..c4dda76e86 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -10,6 +10,7 @@ + diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs index 414a1009aa..1d5ed54bff 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProviderTests.cs @@ -21,7 +21,8 @@ public void TestImportInternalCataloguesOnly() var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2"); _cata1.SaveToDatabase(); _cata2.SaveToDatabase(); - + var ext = new ExtractableDataSet(GetMockActivator().RepositoryLocator.DataExportRepository, _cata2); + ext.SaveToDatabase(); var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test",1,"test"); var provider = new InternalDatasetProvider(GetMockActivator(),configuration,null); var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false,true, false, false,false); @@ -52,7 +53,8 @@ public void TestImportDeprecatedCataloguesOnly() var _cata2 = new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "Dataset2"); _cata1.SaveToDatabase(); _cata2.SaveToDatabase(); - + var ext = new ExtractableDataSet(GetMockActivator().RepositoryLocator.DataExportRepository, _cata2); + ext.SaveToDatabase(); var configuration = new DatasetProviderConfiguration(GetMockActivator().RepositoryLocator.CatalogueRepository, "test", "test", "test", 1, "test"); var provider = new InternalDatasetProvider(GetMockActivator(), configuration, null); var cmd = new ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(GetMockActivator(), provider, false, false, false, true, false); diff --git a/Rdmp.Core.Tests/Datasets/Jira/JiraDatasetProviderTests.cs b/Rdmp.Core.Tests/Datasets/Jira/JiraDatasetProviderTests.cs new file mode 100644 index 0000000000..1b673f5c20 --- /dev/null +++ b/Rdmp.Core.Tests/Datasets/Jira/JiraDatasetProviderTests.cs @@ -0,0 +1,334 @@ +using Amazon.Runtime.Internal.Endpoints.StandardLibrary; +using JustEat.HttpClientInterception; +using NUnit.Framework; +using Org.BouncyCastle.Asn1.X509; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data.Datasets.Jira; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using Tests.Common; +using static System.Net.WebRequestMethods; + +namespace Rdmp.Core.Tests.Datasets.Jira; + +class JiraDatasetProviderTests : DatabaseTests +{ + + private DatasetProviderConfiguration _configuration; + private JiraDatasetProvider _provider; + private Catalogue _catalogue; + private HttpClient _mockHttp; + private HttpClientInterceptorOptions options = new HttpClientInterceptorOptions { ThrowOnMissingRegistration = true }; + + [TearDown] + public void OneTimeTeardown() + { + _mockHttp.Dispose(); + } + + [OneTimeSetUp] + public void OneTimeSetup() + { + var dataAccessCredentials = new DataAccessCredentials(CatalogueRepository, "name") + { + Username = "test", + Password = "test" + }; + dataAccessCredentials.SaveToDatabase(); + + _configuration = new DatasetProviderConfiguration(CatalogueRepository, "Test Provider", typeof(JiraDatasetProvider).ToString(), "4", dataAccessCredentials.ID, "1234"); + + _configuration.SaveToDatabase(); + + _provider = new JiraDatasetProvider(new ThrowImmediatelyActivator(RepositoryLocator), _configuration, _mockHttp); + _catalogue = new Catalogue(CatalogueRepository, "Jira test Catalogue"); + _catalogue.SaveToDatabase(); + + } + + private void SetupHttpClient(HttpRequestInterceptionBuilder builder) + { + builder.RegisterWith(options); + _mockHttp = options.CreateHttpClient(); + _provider = new JiraDatasetProvider(new ThrowImmediatelyActivator(RepositoryLocator), _configuration, _mockHttp); + } + + [Test] + public void CreateProviderTest() + { + Assert.DoesNotThrow(() => _configuration.GetProviderInstance(new ThrowImmediatelyActivator(RepositoryLocator))); + } + + //AddExistingDataset + [Test] + public void AddExistingDatasetTest_Success() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/{url}") + .Responds() + .WithStatus(200).WithJsonContent(SucessResponseString); + SetupHttpClient(builder); + string name = "AddExistingDatasetTest_Success"; + Assert.DoesNotThrow(() => _provider.AddExistingDataset(name, url)); + var dataset = CatalogueRepository.GetAllObjects().FirstOrDefault(d => d.Name == name); + Assert.That(dataset, Is.Not.Null); + dataset.DeleteInDatabase(); + } + [Test] + public void AddExistingDatasetTest_BadRemote() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/{url}") + .Responds() + .WithStatus(400); + SetupHttpClient(builder); + string name = "AddExistingDatasetTest_BadRemote"; + Assert.Throws(() => _provider.AddExistingDataset(name, url)); + var dataset = CatalogueRepository.GetAllObjects().FirstOrDefault(d => d.Name == name); + Assert.That(dataset, Is.Null); + } + //AddExistingDatasetWithReturn + [Test] + public void AddExistingDatasetWithReturnTest_Success() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/{url}") + .Responds() + .WithStatus(200).WithJsonContent(SucessResponseString); + SetupHttpClient(builder); + string name = "AddExistingDatasetWithReturnTest_Success"; + Assert.DoesNotThrow(() => _provider.AddExistingDatasetWithReturn(name, url)); + var dataset = CatalogueRepository.GetAllObjects().FirstOrDefault(d => d.Name == name); + Assert.That(dataset, Is.Not.Null); + dataset.DeleteInDatabase(); + } + [Test] + public void AddExistingDatasetWithReturnTest_BadRemote() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/{url}") + .Responds() + .WithStatus(400); + SetupHttpClient(builder); + string name = "AddExistingDatasetWithReturnTest_BadRemote"; + Assert.Throws(() => _provider.AddExistingDatasetWithReturn(name, url)); + var dataset = CatalogueRepository.GetAllObjects().FirstOrDefault(d => d.Name == name); + Assert.That(dataset, Is.Null); + } + + [Test] + public void CreateDataset_BadObjectType() + { + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/objectschema/4/objecttypes") + .Responds() + .WithStatus(400); + SetupHttpClient(builder); + + builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForPost() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/create") + .Responds() + .WithStatus(201).WithJsonContent(SucessCreateString); + SetupHttpClient(builder); + + builder = new HttpRequestInterceptionBuilder() + .Requests().ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/objecttype/1/attributes") + .Responds() + .WithStatus(200).WithJsonContent(SuccessObjectTypeAttributes); + SetupHttpClient(builder); + + builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/999") + .Responds() + .WithStatus(200).WithJsonContent(SucessResponseString); + SetupHttpClient(builder); + + var catalogue = new Catalogue(CatalogueRepository, "CreateDataset_Success"); + catalogue.SaveToDatabase(); + var e = Assert.Throws(() => _provider.Create(catalogue)); + Assert.That(e.Message, Is.EqualTo("BadRequest: Unable to fetch Object Types")); + } + + [Test] + public void CreateDataset_UnableToCreate() + { + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/objectschema/4/objecttypes") + .Responds() + .WithStatus(200).WithJsonContent(SucessObjectTypesString); + SetupHttpClient(builder); + + builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForPost() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/create") + .Responds() + .WithStatus(400); + SetupHttpClient(builder); + + builder = new HttpRequestInterceptionBuilder() + .Requests().ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/objecttype/1/attributes") + .Responds() + .WithStatus(200).WithJsonContent(SuccessObjectTypeAttributes); + SetupHttpClient(builder); + + builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/999") + .Responds() + .WithStatus(200).WithJsonContent(SucessResponseString); + SetupHttpClient(builder); + + var catalogue = new Catalogue(CatalogueRepository, "CreateDataset_Success"); + catalogue.SaveToDatabase(); + var e = Assert.Throws(() => _provider.Create(catalogue)); + Assert.That(e.Message, Is.EqualTo("BadRequest: Unable to create Dataset")); + } + + [Test] + public void CreateDataset_NoMatchingSchema() + { + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/objectschema/4/objecttypes") + .Responds() + .WithStatus(200).WithJsonContent(SucessObjectTypesString); + SetupHttpClient(builder); + + builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForPost() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/create") + .Responds() + .WithStatus(201).WithJsonContent(SucessCreateString); + SetupHttpClient(builder); + + builder = new HttpRequestInterceptionBuilder() + .Requests().ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/objecttype/1/attributes") + .Responds() + .WithStatus(400); + SetupHttpClient(builder); + + builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/999") + .Responds() + .WithStatus(200).WithJsonContent(SucessResponseString); + SetupHttpClient(builder); + + var catalogue = new Catalogue(CatalogueRepository, "CreateDataset_Success"); + catalogue.SaveToDatabase(); + var e = Assert.Throws(() => _provider.Create(catalogue)); + Assert.That(e.Message, Is.EqualTo("BadRequest: Unable to fetch Object Attributes for Schema 4")); + } + //FetchDatasetByID + [Test] + public void FetchDatasetByIDTest_Success() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/{url}") + .Responds() + .WithStatus(200).WithJsonContent(SucessResponseString); + SetupHttpClient(builder); + Assert.DoesNotThrow(() => _provider.FetchDatasetByID(int.Parse(url))); + } + [Test] + public void FetchDatasetByIDTest_BadRemote() + { + var url = "999"; + var builder = new HttpRequestInterceptionBuilder() + .Requests() + .ForHttps() + .ForHost("api.atlassian.com/jsm/assets/workspace") + .ForPath($"/1234/v1/object/{url}") + .Responds() + .WithStatus(400); + SetupHttpClient(builder); + Assert.Throws(() => _provider.FetchDatasetByID(int.Parse(url))); + } + + private object SucessResponseString = new + { + id = "22", + name = "AddExistingDatasetWithReturnTest_Success", + type = "Dataset", + url = "22", + source = "Jira", + _links = new + { + self = "https://api.atlassian.com/jsm/assets/workspace/1234/v1/object/22", + } + }; + + private object SucessCreateString = new + { + + }; + + private List SucessObjectTypesString = [new { name = "Dataset", id = 1, objectSchemaId = 4 }]; + + private List SuccessObjectTypeAttributes = [ + new {name="Name",id=2 } + ]; + + private object SucessAQLResult = new + { + values = new List() + }; +} + + + diff --git a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj index 4821571fde..c449d5feca 100644 --- a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj +++ b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj @@ -68,6 +68,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs index a3a572722d..ba2bad5388 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs @@ -60,6 +60,7 @@ public override void Execute() foreach (var catalogue in catalogues) { var dataset = _provider.Create(catalogue); + var ds = _provider.AddExistingDatasetWithReturn(null, dataset.GetID()); var cmd = new ExecuteCommandLinkCatalogueToDataset(_activator, catalogue, ds,_autoUpdate); cmd.Execute(); diff --git a/Rdmp.Core/Curation/Data/Datasets/IDataset.cs b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs index 3b73096298..a9cfcc1ce1 100644 --- a/Rdmp.Core/Curation/Data/Datasets/IDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/IDataset.cs @@ -35,6 +35,7 @@ public interface IDataset: IMapsDirectlyToDatabaseTable /// string Type { get; } + /// /// The URL to access the dataset /// @@ -42,4 +43,5 @@ public interface IDataset: IMapsDirectlyToDatabaseTable public abstract string GetID(); public abstract string GetRemoteID(); + } diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/API/AQLResult.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/API/AQLResult.cs new file mode 100644 index 0000000000..c92620a38a --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/API/AQLResult.cs @@ -0,0 +1,24 @@ +using Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.API +{ + /// + /// + /// + public class AQLResult + { + public int startAt { get; set; } + public int maxResults { get; set; } + public int total { get; set; } + public List values { get; set; } + public List objectTypeAttributes { get; set; } + public bool hasMoreResults { get; set; } + public bool last { get; set; } + public bool isLast { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/API/Avatar.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/API/Avatar.cs new file mode 100644 index 0000000000..a8e80ef356 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/API/Avatar.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.API +{ + /// + /// + /// + public class Avatar + { + public string workspaceId { get; set; } + public string url16 { get; set; } + public string url48 { get; set; } + public string url72 { get; set; } + public string url144 { get; set; } + public string url288 { get; set; } + public string objectId { get; set; } + public MediaClientConfig mediaClientConfig { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/API/Entry.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/API/Entry.cs new file mode 100644 index 0000000000..3b98ec2dfb --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/API/Entry.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.API +{ + /// + /// + /// + public class Entry + { + public string workspaceId { get; set; } + public string globalId { get; set; } + public string id { get; set; } + public string name { get; set; } + public int position { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + public int objectCount { get; set; } + public string objectSchemaId { get; set; } + public bool inherited { get; set; } + public bool abstractObjectType { get; set; } + public bool parentObjectTypeInherited { get; set; } + public string description { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/API/MediaClientConfig.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/API/MediaClientConfig.cs new file mode 100644 index 0000000000..4515c3ae90 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/API/MediaClientConfig.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.API +{ + /// + /// + /// + public class MediaClientConfig + { + public string clientId { get; set; } + public string issuer { get; set; } + public string mediaBaseUrl { get; set; } + public string mediaJwtToken { get; set; } + public string fileId { get; set; } + public int tokenLifespanInMinutes { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/API/UpdateAttributes.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/API/UpdateAttributes.cs new file mode 100644 index 0000000000..1801fb9e81 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/API/UpdateAttributes.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.API +{ + /// + /// + /// + public class UpdateAttributes + { + public List attributes { get; set; } + + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/API/Value.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/API/Value.cs new file mode 100644 index 0000000000..87121f9ac6 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/API/Value.cs @@ -0,0 +1,31 @@ +using Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.API +{ + /// + /// + /// + public class Value + { + public string workspaceId { get; set; } + public string globalId { get; set; } + public string id { get; set; } + public string label { get; set; } + public string objectKey { get; set; } + public Avatar avatar { get; set; } + public ObjectType objectType { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + public bool hasAvatar { get; set; } + public long timestamp { get; set; } + public List attributes { get; set; } + public Links _links { get; set; } + public string name { get; set; } + } + +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDataset.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDataset.cs new file mode 100644 index 0000000000..800098cba9 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDataset.cs @@ -0,0 +1,45 @@ +using Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects; +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira +{ + /// + /// Mapping of Jira API Asset object to C# + /// + public class JiraDataset : PluginDataset + { + public JiraDataset() + { + } + public JiraDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name) + { + } + + public override string GetID() + { + return id; + } + public override string GetRemoteID() + { + return _links.self.Split("/").Last(); + } + + public string workspaceId { get; set; } + public string globalId { get; set; } + public string id { get; set; } + public string label { get; set; } + public string objectKey { get; set; } + public ObjectType objectType { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + public List attributes { get; set; } + public ExtendedInfo extendedInfo { get; set; } + public Links _links { get; set; } + public string name { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/Attribute.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/Attribute.cs new file mode 100644 index 0000000000..0581cfb398 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/Attribute.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class Attribute + { + public string workspaceId { get; set; } + public string globalId { get; set; } + public string id { get; set; } + public ObjectTypeAttribute objectTypeAttribute { get; set; } + public string objectTypeAttributeId { get; set; } + public List objectAttributeValues { get; set; } + public string objectId { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/DefaultType.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/DefaultType.cs new file mode 100644 index 0000000000..f2ca93c449 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/DefaultType.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class DefaultType + { + public int id { get; set; } + public string name { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ExtendedInfo.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ExtendedInfo.cs new file mode 100644 index 0000000000..d3787f5c71 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ExtendedInfo.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class ExtendedInfo + { + public bool openIssuesExists { get; set; } + public bool attachmentsExists { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/Icon.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/Icon.cs new file mode 100644 index 0000000000..9b1b02d698 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/Icon.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class Icon + { + public string id { get; set; } + public string name { get; set; } + public string url16 { get; set; } + public string url48 { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/Links.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/Links.cs new file mode 100644 index 0000000000..902a72b210 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/Links.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class Links + { + public string self { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ObjectAttributeValue.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ObjectAttributeValue.cs new file mode 100644 index 0000000000..6495cf2fc7 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ObjectAttributeValue.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class ObjectAttributeValue + { + public object value { get; set; } + public object searchValue { get; set; } + public bool referencedType { get; set; } + public string displayValue { get; set; } + public ReferencedObject referencedObject { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ObjectType.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ObjectType.cs new file mode 100644 index 0000000000..452220fec7 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ObjectType.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class ObjectType + { + public string workspaceId { get; set; } + public string globalId { get; set; } + public string id { get; set; } + public string name { get; set; } + public int type { get; set; } + public string description { get; set; } + public Icon icon { get; set; } + public int position { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + public int objectCount { get; set; } + public string objectSchemaId { get; set; } + public bool inherited { get; set; } + public bool abstractObjectType { get; set; } + public bool parentObjectTypeInherited { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ObjectTypeAttribute.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ObjectTypeAttribute.cs new file mode 100644 index 0000000000..87fea6260a --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ObjectTypeAttribute.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class ObjectTypeAttribute + { + public string workspaceId { get; set; } + public string globalId { get; set; } + public string id { get; set; } + public string name { get; set; } + public bool label { get; set; } + public int type { get; set; } + public DefaultType defaultType { get; set; } + public bool editable { get; set; } + public bool system { get; set; } + public bool sortable { get; set; } + public bool summable { get; set; } + public bool indexed { get; set; } + public int minimumCardinality { get; set; } + public int maximumCardinality { get; set; } + public bool removable { get; set; } + public bool hidden { get; set; } + public bool includeChildObjectTypes { get; set; } + public bool uniqueAttribute { get; set; } + public string options { get; set; } + public int position { get; set; } + public string description { get; set; } + public ReferenceType referenceType { get; set; } + public string referenceObjectTypeId { get; set; } + public ReferenceObjectType referenceObjectType { get; set; } + public string suffix { get; set; } + public string regexValidation { get; set; } + public string qlQuery { get; set; } + public string iql { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ReferenceObjectType.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ReferenceObjectType.cs new file mode 100644 index 0000000000..09073e48a7 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ReferenceObjectType.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class ReferenceObjectType + { + public string workspaceId { get; set; } + public string globalId { get; set; } + public string id { get; set; } + public string name { get; set; } + public int type { get; set; } + public string description { get; set; } + public Icon icon { get; set; } + public int position { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + public int objectCount { get; set; } + public string objectSchemaId { get; set; } + public bool inherited { get; set; } + public bool abstractObjectType { get; set; } + public bool parentObjectTypeInherited { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ReferenceType.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ReferenceType.cs new file mode 100644 index 0000000000..9a42836615 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ReferenceType.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class ReferenceType + { + public string workspaceId { get; set; } + public string globalId { get; set; } + public string id { get; set; } + public string name { get; set; } + public string description { get; set; } + public string color { get; set; } + public string url16 { get; set; } + public bool removable { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ReferencedObject.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ReferencedObject.cs new file mode 100644 index 0000000000..d147d94cbc --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetObjects/ReferencedObject.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects +{ + /// + /// + /// + public class ReferencedObject + { + public string workspaceId { get; set; } + public string globalId { get; set; } + public string id { get; set; } + public string label { get; set; } + public string objectKey { get; set; } + public ObjectType objectType { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + public bool hasAvatar { get; set; } + public object timestamp { get; set; } + public Links _links { get; set; } + public string name { get; set; } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetProvider.cs b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetProvider.cs new file mode 100644 index 0000000000..f00a1590f9 --- /dev/null +++ b/Rdmp.Core/Curation/Data/Datasets/Jira/JiraDatasetProvider.cs @@ -0,0 +1,413 @@ +using Newtonsoft.Json; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data.Datasets.Jira.API; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data.Datasets.Jira +{ + /// + /// + /// + public class JiraDatasetProvider : PluginDatasetProvider + { + private readonly string _workspace; + private readonly HttpClient _client; + private readonly string API_URL = "https://api.atlassian.com/jsm/assets/workspace/"; + private readonly String DATASET = "Dataset"; + private readonly String PROJECT = "Project"; + private readonly String NAME = "Name"; + + private readonly JsonSerializerOptions serializeOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + IncludeFields = true, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + + public JiraDatasetProvider(IBasicActivateItems activator, DatasetProviderConfiguration configuration, HttpClient httpClient = null) : base(activator, configuration) + { + _client = httpClient ?? new HttpClient(); + var credentials = Repository.GetAllObjectsWhere("ID", Configuration.DataAccessCredentials_ID).First(); + var apiKey = credentials.GetDecryptedPassword(); + var code = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{credentials.Username}:{apiKey}")); + _client.DefaultRequestHeaders.Add("Authorization", $"Basic {code}"); + + _client.DefaultRequestHeaders + .Accept + .Add(new MediaTypeWithQualityHeaderValue("application/json")); + _workspace = configuration.Organisation_ID; + } + + public override void AddExistingDataset(string name, string url) + { + AddExistingDatasetWithReturn(name, url); + } + + public override Dataset AddExistingDatasetWithReturn(string name, string url) + { + JiraDataset jiraDataset = (JiraDataset)FetchDatasetByID(int.Parse(url)); + var dataset = new Dataset(Repository, name??jiraDataset.name) + { + Url = jiraDataset._links.self, + Type = this.ToString(), + Provider_ID = Configuration.ID, + Folder = $"\\Jira\\{Configuration.Name}", + }; + dataset.SaveToDatabase(); + Activator.Publish(dataset); + return dataset; + } + + private class CreateAtrObj + { + public string objectTypeId; + public List Attributes; + } + + public override JiraDataset Create(Catalogue catalogue) + { + var url = $"{API_URL}{_workspace}/v1/object/create"; + using var stream = new MemoryStream(); + + var response = Task.Run(async () => await _client.GetAsync($"{API_URL}{_workspace}/v1/objectschema/{Configuration.Url}/objecttypes")).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + List objectSchemaTypes = JsonConvert.DeserializeObject>(detailsString); + var t = objectSchemaTypes.FirstOrDefault(e => e.name == DATASET); + string objectTypeId = t.id; + string nameAttributeId = ""; + var schema = GetSchemaAttributes(t.objectSchemaId); + var item = schema.Where(s => s.name == NAME); + if (item.Any()) + { + nameAttributeId = item.First().id; + } + else + { + throw new Exception($"{response.StatusCode}: No matching Schema found for {NAME}"); + } + var o = new CreateAtrObj() + { + objectTypeId = objectTypeId, + Attributes = new List() { new JiraDatasetObjects.Attribute() { + objectTypeAttributeId=nameAttributeId, + objectAttributeValues = new List(){ + new(){value=catalogue.Name} + } + } } + }; + + + System.Text.Json.JsonSerializer.Serialize(stream, o, serializeOptions); + var jsonString = Encoding.UTF8.GetString(stream.ToArray()); + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + response = Task.Run(async () => await _client.PostAsync(url, httpContent)).Result; + if (response.StatusCode == HttpStatusCode.Created) + { + detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + JiraDataset jiraDataset = JsonConvert.DeserializeObject(detailsString); + jiraDataset.Url = jiraDataset._links.self; + UpdateUsingCatalogue(jiraDataset, catalogue); + return jiraDataset; + } + else + { + throw new Exception($"{response.StatusCode}: Unable to create Dataset"); + } + } + else + { + throw new Exception($"{response.StatusCode}: Unable to fetch Object Types"); + } + + } + + public override Dataset FetchDatasetByID(int id) + { + var response = Task.Run(async () => await _client.GetAsync($"{API_URL}{_workspace}/v1/object/{id}")).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + JiraDataset jiraDataset = JsonConvert.DeserializeObject(detailsString); + + return jiraDataset; + } + throw new Exception($"{response.StatusCode}: Unable to fetch Dataset by ID {id}"); + } + + public override void Update(string uuid, PluginDataset datasetUpdates) + { + var ds = (JiraDataset)datasetUpdates; + List jiraAttributes = []; + foreach (var attribute in ds.attributes) + { + var obj = new JiraDatasetObjects.Attribute(); + obj.objectTypeAttributeId = attribute.objectTypeAttributeId; + obj.objectAttributeValues = new List(); + foreach (var v in attribute.objectAttributeValues) + { + obj.objectAttributeValues.Add(new JiraDatasetObjects.ObjectAttributeValue() + { + value = v.value + }); + } + jiraAttributes.Add(obj); + } + using var stream = new MemoryStream(); + var o = new UpdateAttributes() + { + attributes = jiraAttributes + }; + System.Text.Json.JsonSerializer.Serialize(stream, o, serializeOptions); + var jsonString = Encoding.UTF8.GetString(stream.ToArray()); + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + var response = Task.Run(async () => await _client.PutAsync($"{API_URL}{_workspace}/v1/object/{uuid}", httpContent)).Result; + if (response.StatusCode != HttpStatusCode.OK) + { + throw new Exception($"{response.StatusCode}: Unable to update Dataset"); + } + } + + private static string GetObjectTypeAttributeID(JiraDataset dataset, string name) + { + try + { + var a = dataset.attributes.Select(a => a.objectTypeAttribute); + var b = a.Select(a => a.name); + var c = a.Where(l => l.name == name).ToList(); + if (c.Any()) return c.First().id; + return null; + } + catch (Exception) + { + return null; + } + } + + + private List _schemaObjects = []; + + private List GetSchemaAttributes(string objectSchemaID) + { + + if (_schemaObjects.Any()) return _schemaObjects; + + var otresponse = Task.Run(async () => await _client.GetAsync($"{API_URL}{_workspace}/v1/objectschema/{objectSchemaID}/objecttypes")).Result; + if (otresponse.StatusCode == HttpStatusCode.OK) + { + var otdetailsString = Task.Run(async () => await otresponse.Content.ReadAsStringAsync()).Result; + var _objectEntires = JsonConvert.DeserializeObject>(otdetailsString); + var o = _objectEntires.FirstOrDefault(o => o.name == DATASET); + if (o is not null) + { + var id = o.id; + var response = Task.Run(async () => await _client.GetAsync($"{API_URL}{_workspace}/v1/objecttype/{id}/attributes")).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + _schemaObjects = JsonConvert.DeserializeObject>(detailsString); + return _schemaObjects; + } + else + { + throw new Exception($"{response.StatusCode}: Unable to fetch Object Attributes for Schema {objectSchemaID}"); + } + } + throw new Exception("Unable to find Dataset object"); + } + else + { + throw new Exception($"{otresponse.StatusCode}: Unable to fetch Object Types"); + + } + + + + } + + private JiraDatasetObjects.Attribute GenerateUpdateAttribute(JiraDataset dataset, Catalogue catalogue, string name, string value) + { + var otai = GetObjectTypeAttributeID(dataset, name); + if (otai is null) + { + var schema = GetSchemaAttributes(dataset.objectType.objectSchemaId); + var item = schema.Where(s => s.name == name); + if (item.Any()) + { + otai = item.First().id; + } + } + return new JiraDatasetObjects.Attribute() + { + objectTypeAttributeId = otai, + objectAttributeValues = new List() { + new JiraDatasetObjects.ObjectAttributeValue() + { + value = value + } + } + }; + } + + public override string GetRemoteURL(Dataset dataset) + { + Activator.Show("Unable to open URL for Jira Assets. Please visit your Atlassian instance to access this Dataset"); + return null; + } + + public override void UpdateUsingCatalogue(Dataset dataset, Catalogue catalogue) + { + var jiraDataset = (JiraDataset)FetchDatasetByID(int.Parse(dataset.Url.Split("/").Last())); + var updateDataset = new JiraDataset(); + updateDataset.attributes = new List() { + GenerateUpdateAttribute(jiraDataset, catalogue, "Name", catalogue.Name), + GenerateUpdateAttribute(jiraDataset, catalogue, "Short Description", catalogue.ShortDescription), + GenerateUpdateAttribute(jiraDataset, catalogue, "Acronym", catalogue.Acronym), + GenerateUpdateAttribute(jiraDataset, catalogue, "DOI", catalogue.Doi), + GenerateUpdateAttribute(jiraDataset, catalogue, "Update Frequency", ((Catalogue.UpdateFrequencies)catalogue.Update_freq).ToString()), + GenerateUpdateAttribute(jiraDataset, catalogue, "Initial Release Date", catalogue.DatasetReleaseDate.ToString()), + GenerateUpdateAttribute(jiraDataset, catalogue, "Update Lag", ((Catalogue.UpdateLagTimes)catalogue.UpdateLag).ToString()), + GenerateUpdateAttribute(jiraDataset, catalogue, "Is Deprecated", catalogue.IsDeprecated.ToString()), + GenerateUpdateAttribute(jiraDataset, catalogue, "Is Project Specific", catalogue.IsProjectSpecific(Activator.RepositoryLocator.DataExportRepository).ToString()), + GenerateUpdateAttribute(jiraDataset, catalogue, "RDMP_CatalogueID", catalogue.ID.ToString()), + GenerateUpdateAttribute(jiraDataset, catalogue, "RDMP_CatalogueDB", (catalogue.CatalogueRepository as TableRepository).GetConnection().Connection.ConnectionString), + }; + + var tableInfos = catalogue.CatalogueItems.Select(ci => ci.ColumnInfo.TableInfo).ToList(); + var databaseTableschema = GetSchemaAttributes(jiraDataset.objectType.objectSchemaId).Where(s => s.name == "Database").First().id; + + var jsonString = "{\r\n \"qlQuery\": \"objectType = Database\"\r\n}"; + var httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + + var response = Task.Run(async () => await _client.PostAsync($"{API_URL}{_workspace}/v1/object/aql", httpContent)).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + AQLResult databases = JsonConvert.DeserializeObject(detailsString); + + var dbUpdate = new JiraDatasetObjects.Attribute(); + dbUpdate.objectId = jiraDataset.id; + dbUpdate.objectTypeAttributeId = databaseTableschema; + dbUpdate.objectAttributeValues = databases.values.Select(d => new JiraDatasetObjects.ObjectAttributeValue() { value = d.objectKey }).ToList(); + + + var dbs = new List(); + foreach (var ti in tableInfos) + { + var o = databases.values.FirstOrDefault(db => db.attributes.Any(a => a.objectAttributeValues.First().value.ToString() == ti.Database[1..^1]) && db.attributes.Any(a => a.objectAttributeValues.First().value.ToString() == ti.Server)); + if (o is not null) + { + dbs.Add(o); + } + } + dbs = dbs.Distinct().ToList(); + + + using var stream = new MemoryStream(); + System.Text.Json.JsonSerializer.Serialize(stream, dbUpdate, serializeOptions); + var dbUpdatejson = Encoding.UTF8.GetString(stream.ToArray()); + + updateDataset.attributes.Add(new JiraDatasetObjects.Attribute() + { + objectTypeAttributeId = databaseTableschema, + objectAttributeValues = dbs.Select(v => new JiraDatasetObjects.ObjectAttributeValue() { value = v.objectKey }).ToList() + }); + } + else + { + throw new Exception($"{response.StatusCode}: Unable to fetch Jira Database Object"); + } + + Update(jiraDataset.GetRemoteID(), updateDataset); + + //update projects + var projectSpecificIDs = Activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", catalogue.ID).Where(eds => eds.Projects != null).SelectMany(eds => eds.Projects).Select(p => p.ID).Distinct(); + var projectSpecifics = Activator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(p => projectSpecificIDs.Contains(p.ID)); + var projectsUsedIn = Activator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(p => p.ExtractionConfigurations.Any(ec => ec.GetAllExtractableDataSets().Any(eds => eds.Catalogue_ID == catalogue.ID))); + var linkedProjects = projectSpecifics.Concat(projectsUsedIn).ToList().Distinct(); + + response = Task.Run(async () => await _client.GetAsync($"{API_URL}{_workspace}/v1/objectschema/{Configuration.Url}/objecttypes")).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + var detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + List objectSchemaTypes = JsonConvert.DeserializeObject>(detailsString); + var t = objectSchemaTypes.FirstOrDefault(e => e.name == PROJECT); + + + + foreach (var project in linkedProjects) + { + var catalogues = project.GetAllProjectCatalogues();//is this all of them? + List datasets = catalogue.GetLinkedDatasets().Where(ds => ds.Provider_ID == Configuration.ID).ToList(); + + + + jsonString = $"{{\r\n \"qlQuery\": \"objectType = \\\"Project - This should have been a CM\\\" and \\\"Project ID\\\" startswith \\\"Project {project.ProjectNumber} \\\"\"\r\n}}"; + httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + response = Task.Run(async () => await _client.PostAsync($"{API_URL}{_workspace}/v1/object/aql", httpContent)).Result; + if (response.StatusCode == HttpStatusCode.OK) + { + + detailsString = Task.Run(async () => await response.Content.ReadAsStringAsync()).Result; + AQLResult projects = JsonConvert.DeserializeObject(detailsString); + if (projects.objectTypeAttributes.Any()) + { + var datasetTypeAttribute = projects.objectTypeAttributes.FirstOrDefault(ota => ota.name == DATASET); + if (datasetTypeAttribute is not null) + { + var datasetID = datasetTypeAttribute.id; + + foreach (var jiraAssetProject in projects.values) + { + List jiraAttributes = []; + jiraAttributes.Add(new JiraDatasetObjects.Attribute() + { + objectTypeAttributeId = datasetID, + objectAttributeValues = datasets.Select(ds => new JiraDatasetObjects.ObjectAttributeValue() { value = ((JiraDataset)FetchDatasetByID(int.Parse(ds.Url.Split('/').Last()))).objectKey }).ToList() + + }); + using var stream = new MemoryStream(); + var o = new UpdateAttributes() + { + attributes = jiraAttributes + }; + System.Text.Json.JsonSerializer.Serialize(stream, o, serializeOptions); + jsonString = Encoding.UTF8.GetString(stream.ToArray()); + httpContent = new StringContent(jsonString, Encoding.UTF8, "application/json"); + response = Task.Run(async () => await _client.PutAsync($"{API_URL}{_workspace}/v1/object/{jiraAssetProject.id}", httpContent)).Result; + if (response.StatusCode != HttpStatusCode.OK) + { + throw new Exception($"{response.StatusCode}: Unable to Link Dataset to project {jiraAssetProject.id}"); + } + } + } + } + } + else + { + throw new Exception($"{response.StatusCode}: Unable to fetch Projects"); + } + } + } + else + { + throw new Exception($"{response.StatusCode}: Unable to fetch Object types"); + } + } + } +} diff --git a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs index 887b68e403..06f8e0eca4 100644 --- a/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs +++ b/Rdmp.Core/Curation/Data/Datasets/PluginDataset.cs @@ -9,7 +9,7 @@ namespace Rdmp.Core.Curation.Data.Datasets /// /// Base class to allow all plugin dataset types to be based off /// - public class PluginDataset : Rdmp.Core.Curation.Data.Datasets.Dataset + public class PluginDataset : Dataset { public PluginDataset(ICatalogueRepository catalogueRepository, string name) : base(catalogueRepository, name) { } public PluginDataset() { } diff --git a/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs b/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs index 2585a5baf8..9dfbb7af4c 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs @@ -29,7 +29,8 @@ public static void FindProblems(List csFilesFound) "PluginPackagerProgramOptions.cs", //allowed because it's a suggestion to the user about command line arguments "DocumentationCrossExaminationTest.cs", //allowed because its basically a list of comments that are allowed despite not appearing in the codebase "ResearchDataManagementPlatformOptions.cs", //allowed because it's an Example - "AWSS3BucketReleaseDestination.cs" //allowed as it uses it as a temp file identifier + "AWSS3BucketReleaseDestination.cs", //allowed as it uses it as a temp file identifier + "JiraDatasetProvider.cs" //Jira attribute uses the prohibited string }; diff --git a/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs index bd07409e71..6e72520a72 100644 --- a/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs +++ b/Rdmp.UI/SubComponents/CreateExternalDatasetDialog.cs @@ -10,6 +10,7 @@ using System.Text; using System.Threading.Tasks; using System.Windows.Forms; +using System.Security.Policy; namespace Rdmp.UI.SubComponents { diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index 6e3b082d47..ace612cd5c 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -29,6 +29,8 @@ using Rdmp.Core.Curation.Data.Dashboarding; using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.Curation.Data.Datasets; +using Rdmp.Core.Curation.Data.Datasets.Jira; +using Rdmp.Core.Curation.Data.Datasets.Jira.JiraDatasetObjects; using Rdmp.Core.Curation.Data.Governance; using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Pipelines; @@ -622,7 +624,11 @@ public static T WhenIHaveA(MemoryDataExportRepository repository) where T : D { return (T)(object)new DatasetProviderConfiguration(repository.CatalogueRepository, "","","",WhenIHaveA(repository).ID,""); } - if(typeof(T) == typeof(ExtractableDataSetProject)) + if(typeof(T) == typeof(JiraDataset)) + { + return (T)(object)new JiraDataset(repository.CatalogueRepository,"Jira Dataset"); + } + if(typeof(T) == typeof(ExtractableDataSetProject)) { return (T)(object)new ExtractableDataSetProject(repository, WhenIHaveA(repository), WhenIHaveA(repository)); }