diff --git a/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj index 66f31c1..1b7895d 100644 --- a/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj @@ -1,6 +1,6 @@  - 10.8.4 + 10.9.0 1.0.0.0 OData Provider The Odata Provider lets you fetch and map data from or to any OData endpoint. @@ -24,11 +24,15 @@ snupkg - + + + ..\..\..\Dynamicweb10\src\Features\DataIntegration\Dynamicweb.DataIntegration\bin\Debug\net8.0\Dynamicweb.DataIntegration.dll + + diff --git a/src/ODataProvider.cs b/src/ODataProvider.cs index a949df5..d96968c 100644 --- a/src/ODataProvider.cs +++ b/src/ODataProvider.cs @@ -1,25 +1,27 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using System.Xml; -using System.Xml.Linq; -using Dynamicweb.Core; +using Dynamicweb.Core; using Dynamicweb.DataIntegration.EndpointManagement; using Dynamicweb.DataIntegration.Integration; using Dynamicweb.DataIntegration.Integration.ERPIntegration; using Dynamicweb.DataIntegration.Integration.Interfaces; +using Dynamicweb.DataIntegration.ProviderHelpers; using Dynamicweb.DataIntegration.Providers.ODataProvider.Interfaces; using Dynamicweb.DataIntegration.Providers.ODataProvider.Model; using Dynamicweb.Extensibility.AddIns; using Dynamicweb.Extensibility.Editors; using Dynamicweb.Logging; using Dynamicweb.Security.Licensing; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Xml; +using System.Xml.Linq; namespace Dynamicweb.DataIntegration.Providers.ODataProvider; @@ -31,7 +33,7 @@ namespace Dynamicweb.DataIntegration.Providers.ODataProvider; [ResponseMapping(true)] public class ODataProvider : BaseProvider, ISource, IDestination, IParameterOptions, IODataBaseProvider, IParameterVisibility { - internal readonly EndpointService _endpointService = new(); + internal readonly EndpointService _endpointService = new(); internal readonly EndpointCollectionService _endpointCollectionService = new EndpointCollectionService(); internal Schema _schema; internal Endpoint _endpoint; @@ -219,20 +221,20 @@ IEnumerable IParameterOptions.GetParameterOptions(string parame { var result = new List(); - foreach(var collection in _endpointCollectionService.GetEndpointCollections().OrderBy(ec => ec.Sorting)) - { - var parameterOptions = _endpointCollectionService.GetEndpoints(collection.Id).Select(endpoint => - new ParameterOption(endpoint.Name,new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, collection.Name, endpoint.Id.ToString())) - { + foreach (var collection in _endpointCollectionService.GetEndpointCollections().OrderBy(ec => ec.Sorting)) + { + var parameterOptions = _endpointCollectionService.GetEndpoints(collection.Id).Select(endpoint => + new ParameterOption(endpoint.Name, new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, collection.Name, endpoint.Id.ToString())) + { Group = collection.Name }); - result.AddRange(parameterOptions); - } + result.AddRange(parameterOptions); + } result.AddRange(_endpointService.GetEndpoints().Where(e => e.Collection == null).Select(endpoint => - new ParameterOption(endpoint.Name, new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, "Dynamicweb 9 Endpoints", endpoint.Id.ToString())) - { - Group = "Dynamicweb 9 Endpoints" - })); + new ParameterOption(endpoint.Name, new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, "Dynamicweb 9 Endpoints", endpoint.Id.ToString())) + { + Group = "Dynamicweb 9 Endpoints" + })); return result; } @@ -302,34 +304,36 @@ public override void OverwriteDestinationSchemaToOriginal() /// public override Schema GetOriginalSourceSchema() { - var name = GetEntityName(); var entityTypeTables = new Schema(); var entitySetsTables = new Schema(); + if (_endpoint == null) + { + return new Schema(); + } + + var name = GetEntityName(); var header = new Dictionary { { "accept", "text/html,application/xhtml+xml,application/xml" }, { "Content-Type", "text/html" } }; - if (_endpoint != null) + var endpointAuthentication = _endpoint.Authentication; + if (endpointAuthentication != null) { - var endpointAuthentication = _endpoint.Authentication; - if (endpointAuthentication != null) - { - SetCredentials(); - } - Task metadataResponse; - if (endpointAuthentication.IsTokenBased()) - { - string token = OAuthHelper.GetToken(_endpoint, endpointAuthentication); - metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, token); - } - else - { - metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, endpointAuthentication, header); - } - metadataResponse.Wait(); + SetCredentials(); } + Task metadataResponse; + if (endpointAuthentication.IsTokenBased()) + { + string token = OAuthHelper.GetToken(_endpoint, endpointAuthentication); + metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, token); + } + else + { + metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, endpointAuthentication, header); + } + metadataResponse.Wait(); var emptySchema = new Schema(); if (entitySetsTables == emptySchema) @@ -356,9 +360,13 @@ void HandleStream(Stream responseStream, HttpStatusCode responseStatusCode, Dict else if (xmlReader.NodeType == XmlNodeType.Element && xmlReader.Name.Equals("EntitySet", StringComparison.OrdinalIgnoreCase)) { - GetColumnsFromEntityTypeTableToEntitySetTable(entitySetsTables.AddTable(xmlReader.GetAttribute("Name")), entityTypeTables, xmlReader.GetAttribute("EntityType")); + var entityTypeName = xmlReader.GetAttribute("EntityType"); + var SqlSchema = entityTypeName.Substring(entityTypeName.LastIndexOf(".") + 1); + var setTable = entitySetsTables.AddTable(xmlReader.GetAttribute("Name"), SqlSchema); } } + + GetColumnsFromEntityTypeTableToEntitySetTable(entityTypeTables, entitySetsTables); if (!EndpointIsLoadAllEntities(_endpoint.Url)) { var singleEntitySetSelected = entitySetsTables.GetTables().FirstOrDefault(obj => obj.Name.Equals(name, StringComparison.OrdinalIgnoreCase)); @@ -377,20 +385,39 @@ void HandleStream(Stream responseStream, HttpStatusCode responseStatusCode, Dict } } - private void GetColumnsFromEntityTypeTableToEntitySetTable(Table table, Schema entityTypeSchema, string entityTypeName) + private void GetColumnsFromEntityTypeTableToEntitySetTable(Schema entityTypeSchema, Schema entitySetsTables) { - var entityTypeNameClean = entityTypeName.Substring(entityTypeName.LastIndexOf(".") + 1); - Table result = entityTypeSchema.GetTables().Where(obj => obj.Name.Equals(entityTypeNameClean, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); - if (result != null) + var entityTypeSchemaTables = entityTypeSchema.GetTables(); + var entitySetSchemaTables = entitySetsTables.GetTables(); + foreach (var table in entitySetSchemaTables) { - foreach (var item in result.Columns) + Table result = entityTypeSchemaTables.Where(obj => obj.Name.Equals(table.SqlSchema, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + if (result != null) { - if (table.Columns.Where(obj => obj.Name == item.Name).Count() == 0) + foreach (var item in result.Columns) { - table.AddColumn(new Column(item.Name, item.Type, table, item.IsPrimaryKey, item.IsNew, item.ReadOnly)); + if (table.Columns.Where(obj => obj.Name == item.Name).Count() == 0) + { + if (item is TableColumn tableColumn) + { + var tableGroupName = tableColumn.Group; + var columns = entityTypeSchemaTables.FirstOrDefault(obj => obj.Name.Equals(tableColumn.Group, StringComparison.OrdinalIgnoreCase))?.Columns ?? []; + var entitySetTableName = entitySetSchemaTables.FirstOrDefault(obj => obj.SqlSchema.Equals(tableColumn.Group)); + if (entitySetTableName != null) + { + tableGroupName = entitySetTableName.Name; + } + table.AddColumn(new TableColumn(tableColumn.Name, tableGroupName, table, tableColumn.Type, columns)); + } + else + { + table.AddColumn(new Column(item.Name, item.Type, table, item.IsPrimaryKey, item.IsNew, item.ReadOnly)); + } + } } } } + entitySetSchemaTables.ForEach(obj => obj.SqlSchema = string.Empty); } private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table, Schema result) @@ -399,6 +426,7 @@ private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table, string entityName = xmlReader.GetAttribute("Name"); List primaryKeys = new List(); Column column = null; + TableColumn tableColumn = null; while (xmlReader.Read() && !(xmlReader.NodeType == XmlNodeType.EndElement && xmlReader.Name.Equals("EntityType", StringComparison.OrdinalIgnoreCase))) { if (xmlReader.NodeType == XmlNodeType.Element && xmlReader.Name.Equals("PropertyRef", StringComparison.OrdinalIgnoreCase)) @@ -429,6 +457,16 @@ private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table, if (!string.IsNullOrEmpty(permission) && permission.ToLower().EndsWith("permissiontype/read")) column.ReadOnly = true; } + else if (xmlReader.NodeType == XmlNodeType.Element && xmlReader.Name.Equals("NavigationProperty", StringComparison.OrdinalIgnoreCase)) + { + //var containsTarget = xmlReader.GetAttribute("ContainsTarget"); + var navigationPropertyName = xmlReader.GetAttribute("Name"); + var navigationPropertyTypeString = xmlReader.GetAttribute("Type"); + var navigationPropertyType = GetColumnTableType(navigationPropertyTypeString); + var groupName = GetTableName(navigationPropertyTypeString); + tableColumn = new TableColumn(navigationPropertyName, groupName, table, navigationPropertyType, []); + table.AddTableColumn(tableColumn); + } else if (xmlReader.Name.Equals("EntityType", StringComparison.OrdinalIgnoreCase) && xmlReader.GetAttribute("Name") != entityName) { break; @@ -480,6 +518,22 @@ private static Type GetColumnType(string columnTypeString) return typeof(object); } + private static Type GetColumnTableType(string columnTableTypeString) + { + if (columnTableTypeString.StartsWith("Collection", StringComparison.OrdinalIgnoreCase)) + return typeof(Collection); + + return typeof(object); + } + + private static string GetTableName(string columnTableTypeString) + { + var result = columnTableTypeString.Replace("Collection(", "", StringComparison.OrdinalIgnoreCase); + result = result.Replace("Microsoft.NAV.", "", StringComparison.OrdinalIgnoreCase); + result = result.Replace("NAV.", "", StringComparison.OrdinalIgnoreCase); + return result.Replace(")", "", StringComparison.OrdinalIgnoreCase); + } + /// public override ISourceReader GetReader(Mapping mapping) { diff --git a/src/ODataSourceReader.cs b/src/ODataSourceReader.cs index 081ec86..0379b25 100644 --- a/src/ODataSourceReader.cs +++ b/src/ODataSourceReader.cs @@ -166,21 +166,30 @@ internal void CallEndpoing(IDictionary headers, bool readFromLas var selectAsParameters = GetSelectAsParameters(readFromLastRequestResponse); var modeAsParemters = GetModeAsParameters(); var filterAsParameters = GetFilterAsParameters(_mapping); + var expandAsParameters = GetExpandAsParameters(_mapping); + if (!string.IsNullOrEmpty(modeAsParemters)) { filterAsParameters.Add(modeAsParemters); } - if (selectAsParameters.Any()) + if (selectAsParameters.Count != 0) { parameters.Add("$select", string.Join(",", selectAsParameters)); } - if (filterAsParameters.Any()) + if (filterAsParameters.Count != 0) { parameters.Add("$filter", string.Join(" and ", filterAsParameters)); } + if (expandAsParameters.Count != 0) + { + //It is possible to use $select for the middle table by just separating select and expand with semicolon: + //Customers?$select=CustomerID&$expand=Orders($select=OrderID;$expand=Order_Details($select=UnitPrice)) + parameters.Add("$expand", string.Join(",", expandAsParameters)); + } + if (_endpoint.Parameters != null) { foreach (var parameter in _endpoint.Parameters) @@ -336,10 +345,10 @@ private string GetModeAsParameters() private List GetSelectAsParameters(bool readFromLastRequestResponse) { List result = new(); - var activeColumnMappings = _mapping.GetColumnMappings().Where(obj => obj.Active).ToList(); + var activeColumnMappings = _mapping.GetColumnMappings().Where(obj => obj.Active && obj.SourceColumn != null).ToList(); if (activeColumnMappings.Any()) { - var selectColumnNames = activeColumnMappings.Where(obj => obj.SourceColumn != null)?.Select(obj => obj.SourceColumn.Name).ToList(); + var selectColumnNames = activeColumnMappings.Where(obj => string.IsNullOrEmpty(obj.SourceColumn.Group))?.Select(obj => obj.SourceColumn.Name).ToList(); if (readFromLastRequestResponse) { @@ -507,6 +516,17 @@ private void LogWarningForConditional(MappingConditional item) _logger?.Warn($"Can only add {item.ConditionalOperator} on Edm.String and the {item.SourceColumn.Name} is a type of {item.SourceColumn.Type.Name} for the table mapping {_mapping.SourceTable.Name} to {_mapping.DestinationTable.Name}, so this have been removed from the $filter."); } + private List GetExpandAsParameters(Mapping mapping) + { + List result = new(); + var sourceColumnsWithGroups = mapping.SourceTable.Columns.Where(obj => !string.IsNullOrEmpty(obj.Group)); + if (sourceColumnsWithGroups.Any()) + { + result.AddRange(sourceColumnsWithGroups.DistinctBy(obj => obj.Group).Select(obj => obj.Name)); + } + return result; + } + /// /// Handles a specified response stream, by creating an IEnumerable with yield return, so we can later enumerate the result one object at a time without loading them all into memory /// diff --git a/src/ODataWriter.cs b/src/ODataWriter.cs index fee0da7..9d6d2ca 100644 --- a/src/ODataWriter.cs +++ b/src/ODataWriter.cs @@ -31,6 +31,7 @@ internal class ODataWriter : IDisposable, IDestinationWriter public Mapping Mapping { get; } internal JsonObject PostBackObject { get; set; } private readonly ColumnMappingCollection _columnMappings; + private readonly Dictionary _endpointExpand; internal ODataWriter(ILogger logger, Mapping mapping, Endpoint endpoint, ICredentials credentials, bool continueOnError) { @@ -45,12 +46,13 @@ internal ODataWriter(ILogger logger, Mapping mapping, Endpoint endpoint, ICreden _responseMappings = Mapping.GetResponseColumnMappings(); _continueOnError = continueOnError; _columnMappings = Mapping.GetColumnMappings(); + _endpointExpand = GetExpandForMapping(); } public void Write(Dictionary Row) { string endpointURL = Endpoint.Url; - string url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, ""); + string url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, "", _endpointExpand); var keyColumnValuesForFilter = GetKeyColumnValuesForFilter(Row); @@ -69,6 +71,16 @@ public void Write(Dictionary Row) } } } + if (_endpointExpand != null) + { + foreach (var item in _endpointExpand) + { + if (!parameters.ContainsKey(item.Key)) + { + parameters.Add(item.Key, item.Value); + } + } + } url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, "", parameters); var responseFromEndpoint = GetFromEndpoint(url, null); @@ -83,7 +95,7 @@ public void Write(Dictionary Row) var response = responseFromEndpoint?.Result?.Content?.Value; - url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, ""); + url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, "", _endpointExpand); if (response != null && response.Count > 0) { if (response.Count > 1) @@ -133,7 +145,7 @@ public void Write(Dictionary Row) if (primaryKeyColumnValuesForPatch.Any()) { string patchURL = "(" + string.Join(",", primaryKeyColumnValuesForPatch) + ")"; - url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, patchURL); + url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, patchURL, _endpointExpand); } awaitResponseFromEndpoint = PostToEndpoint(url, patchJson, headers, true); } @@ -294,7 +306,9 @@ internal string MapValuesToJson(Dictionary row, bool isPatchRequ { var jsonObject = new JsonObject(); - foreach (ColumnMapping columnMapping in _columnMappings) + var rootMappings = _columnMappings.Where(m => string.IsNullOrEmpty(m.DestinationColumn.Group)); + var nestedMappings = _columnMappings.Where(m => !string.IsNullOrEmpty(m.DestinationColumn.Group)); + foreach (ColumnMapping columnMapping in rootMappings) { if (!columnMapping.Active || (columnMapping.ScriptValueForInsert && isPatchRequest)) continue; @@ -320,12 +334,89 @@ internal string MapValuesToJson(Dictionary row, bool isPatchRequ case "dateonly": jsonObject.Add(columnMapping.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(columnValue, true)); break; + case "guid": + if (string.IsNullOrEmpty(Converter.ToString(columnValue))) + { + jsonObject.Add(columnMapping.DestinationColumn.Name, Converter.ToString(Guid.Empty)); + } + else + { + jsonObject.Add(columnMapping.DestinationColumn.Name, Converter.ToString(columnValue)); + } + break; default: jsonObject.Add(columnMapping.DestinationColumn.Name, Converter.ToString(columnValue)); break; } } } + //not able to patch/update nested objects + if (isPatchRequest) + { + return jsonObject.ToJsonString(); + } + + if (nestedMappings.Any()) + { + var nestedGroups = nestedMappings.DistinctBy(group => group.DestinationColumn.Group).Select(group => group.DestinationColumn.Group); + foreach (var group in nestedGroups) + { + var nestedMapping = nestedMappings.FirstOrDefault(mapping => mapping.DestinationColumn.Group.Equals(group)); + if (nestedMapping == null) + continue; + + if (!row.TryGetValue(nestedMapping.SourceColumn.Group, out var nestedValue)) + { + continue; + } + if (nestedValue is List> nestedValueCollection) + { + var nestedJsonObjectList = new JsonArray(); + foreach (var nestedRow in nestedValueCollection) + { + var nestedJsonObject = new JsonObject(); + foreach (ColumnMapping nestedColumnMapping in nestedMappings.Where(mapping => mapping.DestinationColumn.Group.Equals(group))) + { + var nestedColumnValue = nestedColumnMapping.ConvertInputValueToOutputValue(nestedColumnMapping.HasScriptWithValue ? null : nestedRow.TryGetValue(nestedColumnMapping.SourceColumn?.Name ?? "", out var value) ? value : null); + + switch (nestedColumnMapping.DestinationColumn.Type.Name.ToLower()) + { + case "decimal": + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToDecimal(nestedColumnValue)); + break; + case "int": + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToInt64(nestedColumnValue)); + break; + case "double": + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToDecimal(nestedColumnValue)); + break; + case "datetime": + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(nestedColumnValue, false)); + break; + case "dateonly": + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(nestedColumnValue, true)); + break; + case "guid": + if (string.IsNullOrEmpty(Converter.ToString(nestedColumnValue))) + { + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToString(Guid.Empty)); + } + else + { + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToString(nestedColumnValue)); + } + break; + default: + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToString(nestedColumnValue)); + break; + } + } + nestedJsonObjectList.Add(nestedJsonObject); + } + jsonObject.Add(group, nestedJsonObjectList); + } + } + } return jsonObject.ToJsonString(); } @@ -357,6 +448,24 @@ public static string GetTheDateTimeInZeroTimeZone(object dateTimeObject, bool is return null; } + private Dictionary GetExpandForMapping() + { + if (Mapping.DestinationTable == null) + return []; + + var columnMappings = Mapping.GetColumnMappings(); + + var columnMappingGroups = columnMappings.Where(column => !string.IsNullOrEmpty(column.DestinationColumn.Group)); + if (!columnMappingGroups.Any()) + return []; + + var mappingGroups = columnMappingGroups.DistinctBy(column => column.DestinationColumn.Group).Select(column => column.DestinationColumn.Group).ToList(); + if (mappingGroups.Count != 0) + return new Dictionary() { { "$expand", string.Join(",", mappingGroups) } }; + + return []; + } + public void Dispose() { } public void Close() { }