From 731ded6e9bb018c9ba7aaadc94224e14d5b8da58 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Thu, 16 Jan 2025 10:31:51 +0100 Subject: [PATCH 1/9] added TableColumn logic for saving schema means that when OData table has a NavigationProperty it will be added as one --- ...Integration.Providers.ODataProvider.csproj | 6 +- src/ODataProvider.cs | 120 +++++++++++------- 2 files changed, 82 insertions(+), 44 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj index 66f31c1..74e23e0 100644 --- a/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj @@ -24,11 +24,15 @@ snupkg - + + + ..\..\..\Dynamicweb10\src\Features\DataIntegration\Dynamicweb.DataIntegration\bin\Debug\net8.0\Dynamicweb.DataIntegration.dll + + diff --git a/src/ODataProvider.cs b/src/ODataProvider.cs index a949df5..7b4c577 100644 --- a/src/ODataProvider.cs +++ b/src/ODataProvider.cs @@ -1,25 +1,27 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using System.Xml; -using System.Xml.Linq; -using Dynamicweb.Core; +using Dynamicweb.Core; using Dynamicweb.DataIntegration.EndpointManagement; using Dynamicweb.DataIntegration.Integration; using Dynamicweb.DataIntegration.Integration.ERPIntegration; using Dynamicweb.DataIntegration.Integration.Interfaces; +using Dynamicweb.DataIntegration.ProviderHelpers; using Dynamicweb.DataIntegration.Providers.ODataProvider.Interfaces; using Dynamicweb.DataIntegration.Providers.ODataProvider.Model; using Dynamicweb.Extensibility.AddIns; using Dynamicweb.Extensibility.Editors; using Dynamicweb.Logging; using Dynamicweb.Security.Licensing; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Xml; +using System.Xml.Linq; namespace Dynamicweb.DataIntegration.Providers.ODataProvider; @@ -31,7 +33,7 @@ namespace Dynamicweb.DataIntegration.Providers.ODataProvider; [ResponseMapping(true)] public class ODataProvider : BaseProvider, ISource, IDestination, IParameterOptions, IODataBaseProvider, IParameterVisibility { - internal readonly EndpointService _endpointService = new(); + internal readonly EndpointService _endpointService = new(); internal readonly EndpointCollectionService _endpointCollectionService = new EndpointCollectionService(); internal Schema _schema; internal Endpoint _endpoint; @@ -219,20 +221,20 @@ IEnumerable IParameterOptions.GetParameterOptions(string parame { var result = new List(); - foreach(var collection in _endpointCollectionService.GetEndpointCollections().OrderBy(ec => ec.Sorting)) - { - var parameterOptions = _endpointCollectionService.GetEndpoints(collection.Id).Select(endpoint => - new ParameterOption(endpoint.Name,new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, collection.Name, endpoint.Id.ToString())) - { + foreach (var collection in _endpointCollectionService.GetEndpointCollections().OrderBy(ec => ec.Sorting)) + { + var parameterOptions = _endpointCollectionService.GetEndpoints(collection.Id).Select(endpoint => + new ParameterOption(endpoint.Name, new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, collection.Name, endpoint.Id.ToString())) + { Group = collection.Name }); - result.AddRange(parameterOptions); - } + result.AddRange(parameterOptions); + } result.AddRange(_endpointService.GetEndpoints().Where(e => e.Collection == null).Select(endpoint => - new ParameterOption(endpoint.Name, new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, "Dynamicweb 9 Endpoints", endpoint.Id.ToString())) - { - Group = "Dynamicweb 9 Endpoints" - })); + new ParameterOption(endpoint.Name, new GroupedDropDownParameterEditor.DropDownItem(endpoint.Name, "Dynamicweb 9 Endpoints", endpoint.Id.ToString())) + { + Group = "Dynamicweb 9 Endpoints" + })); return result; } @@ -302,34 +304,36 @@ public override void OverwriteDestinationSchemaToOriginal() /// public override Schema GetOriginalSourceSchema() { - var name = GetEntityName(); var entityTypeTables = new Schema(); var entitySetsTables = new Schema(); + if (_endpoint == null) + { + return new Schema(); + } + + var name = GetEntityName(); var header = new Dictionary { { "accept", "text/html,application/xhtml+xml,application/xml" }, { "Content-Type", "text/html" } }; - if (_endpoint != null) + var endpointAuthentication = _endpoint.Authentication; + if (endpointAuthentication != null) { - var endpointAuthentication = _endpoint.Authentication; - if (endpointAuthentication != null) - { - SetCredentials(); - } - Task metadataResponse; - if (endpointAuthentication.IsTokenBased()) - { - string token = OAuthHelper.GetToken(_endpoint, endpointAuthentication); - metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, token); - } - else - { - metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, endpointAuthentication, header); - } - metadataResponse.Wait(); + SetCredentials(); } + Task metadataResponse; + if (endpointAuthentication.IsTokenBased()) + { + string token = OAuthHelper.GetToken(_endpoint, endpointAuthentication); + metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, token); + } + else + { + metadataResponse = new HttpRestClient(_credentials, 20).GetAsync(GetMetadataURL(), HandleStream, endpointAuthentication, header); + } + metadataResponse.Wait(); var emptySchema = new Schema(); if (entitySetsTables == emptySchema) @@ -387,7 +391,12 @@ private void GetColumnsFromEntityTypeTableToEntitySetTable(Table table, Schema e { if (table.Columns.Where(obj => obj.Name == item.Name).Count() == 0) { - table.AddColumn(new Column(item.Name, item.Type, table, item.IsPrimaryKey, item.IsNew, item.ReadOnly)); + if (item is TableColumn tableColumn) + table.AddColumn(new TableColumn(tableColumn.Name, tableColumn.TableNameReference, table, tableColumn.Type)); + else + { + table.AddColumn(new Column(item.Name, item.Type, table, item.IsPrimaryKey, item.IsNew, item.ReadOnly)); + } } } } @@ -399,6 +408,7 @@ private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table, string entityName = xmlReader.GetAttribute("Name"); List primaryKeys = new List(); Column column = null; + TableColumn tableColumn = null; while (xmlReader.Read() && !(xmlReader.NodeType == XmlNodeType.EndElement && xmlReader.Name.Equals("EntityType", StringComparison.OrdinalIgnoreCase))) { if (xmlReader.NodeType == XmlNodeType.Element && xmlReader.Name.Equals("PropertyRef", StringComparison.OrdinalIgnoreCase)) @@ -429,6 +439,16 @@ private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table, if (!string.IsNullOrEmpty(permission) && permission.ToLower().EndsWith("permissiontype/read")) column.ReadOnly = true; } + else if (xmlReader.NodeType == XmlNodeType.Element && xmlReader.Name.Equals("NavigationProperty", StringComparison.OrdinalIgnoreCase)) + { + //var containsTarget = xmlReader.GetAttribute("ContainsTarget"); + var columnTableName = xmlReader.GetAttribute("Name"); + var columnTableTypeString = xmlReader.GetAttribute("Type"); + var columnTableType = GetColumnTableType(columnTableTypeString); + var tableName = GetTableName(columnTableTypeString); + tableColumn = new TableColumn(columnTableName, tableName, table, columnTableType); + table.AddTableColumn(tableColumn); + } else if (xmlReader.Name.Equals("EntityType", StringComparison.OrdinalIgnoreCase) && xmlReader.GetAttribute("Name") != entityName) { break; @@ -480,6 +500,20 @@ private static Type GetColumnType(string columnTypeString) return typeof(object); } + private static Type GetColumnTableType(string columnTableTypeString) + { + if (columnTableTypeString.StartsWith("Collection", StringComparison.OrdinalIgnoreCase)) + return typeof(Collection); + + return typeof(object); + } + + private static string GetTableName(string columnTableTypeString) + { + var result = columnTableTypeString.Replace("Collection(NAV.", "", StringComparison.OrdinalIgnoreCase); + return result.Replace(")", "", StringComparison.OrdinalIgnoreCase); + } + /// public override ISourceReader GetReader(Mapping mapping) { From 047dcc1b18681751f190e8b87af448ae307a7701 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Mon, 20 Jan 2025 13:05:37 +0100 Subject: [PATCH 2/9] added groupd to TableColumn together with tablecolumn.columns so they are now stored in the schema --- src/ODataProvider.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ODataProvider.cs b/src/ODataProvider.cs index 7b4c577..096f15e 100644 --- a/src/ODataProvider.cs +++ b/src/ODataProvider.cs @@ -384,7 +384,8 @@ void HandleStream(Stream responseStream, HttpStatusCode responseStatusCode, Dict private void GetColumnsFromEntityTypeTableToEntitySetTable(Table table, Schema entityTypeSchema, string entityTypeName) { var entityTypeNameClean = entityTypeName.Substring(entityTypeName.LastIndexOf(".") + 1); - Table result = entityTypeSchema.GetTables().Where(obj => obj.Name.Equals(entityTypeNameClean, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + var entityTypeSchemaTables = entityTypeSchema.GetTables(); + Table result = entityTypeSchemaTables.Where(obj => obj.Name.Equals(entityTypeNameClean, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); if (result != null) { foreach (var item in result.Columns) @@ -392,7 +393,10 @@ private void GetColumnsFromEntityTypeTableToEntitySetTable(Table table, Schema e if (table.Columns.Where(obj => obj.Name == item.Name).Count() == 0) { if (item is TableColumn tableColumn) - table.AddColumn(new TableColumn(tableColumn.Name, tableColumn.TableNameReference, table, tableColumn.Type)); + { + var columns = entityTypeSchemaTables.FirstOrDefault(obj => obj.Name.Equals(tableColumn.TableNameReference,StringComparison.OrdinalIgnoreCase))?.Columns ?? []; + table.AddColumn(new TableColumn(tableColumn.Name, tableColumn.TableNameReference, table, tableColumn.Type, columns)); + } else { table.AddColumn(new Column(item.Name, item.Type, table, item.IsPrimaryKey, item.IsNew, item.ReadOnly)); @@ -446,7 +450,10 @@ private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table, var columnTableTypeString = xmlReader.GetAttribute("Type"); var columnTableType = GetColumnTableType(columnTableTypeString); var tableName = GetTableName(columnTableTypeString); - tableColumn = new TableColumn(columnTableName, tableName, table, columnTableType); + tableColumn = new TableColumn(columnTableName, tableName, table, columnTableType, []) + { + Group = columnTableName + }; table.AddTableColumn(tableColumn); } else if (xmlReader.Name.Equals("EntityType", StringComparison.OrdinalIgnoreCase) && xmlReader.GetAttribute("Name") != entityName) From 49b1026ffb687507de585b1bcb047c44c65a21cb Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Thu, 23 Jan 2025 16:13:38 +0100 Subject: [PATCH 3/9] Work for today --- ...cweb.DataIntegration.Providers.ODataProvider.csproj | 2 +- src/ODataProvider.cs | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj index 74e23e0..1b7895d 100644 --- a/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj @@ -1,6 +1,6 @@  - 10.8.4 + 10.9.0 1.0.0.0 OData Provider The Odata Provider lets you fetch and map data from or to any OData endpoint. diff --git a/src/ODataProvider.cs b/src/ODataProvider.cs index 096f15e..dbe52c2 100644 --- a/src/ODataProvider.cs +++ b/src/ODataProvider.cs @@ -394,8 +394,8 @@ private void GetColumnsFromEntityTypeTableToEntitySetTable(Table table, Schema e { if (item is TableColumn tableColumn) { - var columns = entityTypeSchemaTables.FirstOrDefault(obj => obj.Name.Equals(tableColumn.TableNameReference,StringComparison.OrdinalIgnoreCase))?.Columns ?? []; - table.AddColumn(new TableColumn(tableColumn.Name, tableColumn.TableNameReference, table, tableColumn.Type, columns)); + var columns = entityTypeSchemaTables.FirstOrDefault(obj => obj.Name.Equals(tableColumn.Group,StringComparison.OrdinalIgnoreCase))?.Columns ?? []; + table.AddColumn(new TableColumn(tableColumn.Group, table, tableColumn.Type, columns)); } else { @@ -446,14 +446,10 @@ private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table, else if (xmlReader.NodeType == XmlNodeType.Element && xmlReader.Name.Equals("NavigationProperty", StringComparison.OrdinalIgnoreCase)) { //var containsTarget = xmlReader.GetAttribute("ContainsTarget"); - var columnTableName = xmlReader.GetAttribute("Name"); var columnTableTypeString = xmlReader.GetAttribute("Type"); var columnTableType = GetColumnTableType(columnTableTypeString); var tableName = GetTableName(columnTableTypeString); - tableColumn = new TableColumn(columnTableName, tableName, table, columnTableType, []) - { - Group = columnTableName - }; + tableColumn = new TableColumn(tableName, table, columnTableType, []); table.AddTableColumn(tableColumn); } else if (xmlReader.Name.Equals("EntityType", StringComparison.OrdinalIgnoreCase) && xmlReader.GetAttribute("Name") != entityName) From 618393a43d749fcbc23a76ba53acabd3cfae25a8 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Wed, 29 Jan 2025 09:29:17 +0100 Subject: [PATCH 4/9] added GetExpandAsParameters so it can extend the URL with expand values (tablenames/groups from nested column mappings). ignore nested column mappings in the GetSelectAsParameters for now as this is just a POC --- src/ODataProvider.cs | 11 ++++++----- src/ODataSourceReader.cs | 28 ++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/ODataProvider.cs b/src/ODataProvider.cs index dbe52c2..63b4b7a 100644 --- a/src/ODataProvider.cs +++ b/src/ODataProvider.cs @@ -395,7 +395,7 @@ private void GetColumnsFromEntityTypeTableToEntitySetTable(Table table, Schema e if (item is TableColumn tableColumn) { var columns = entityTypeSchemaTables.FirstOrDefault(obj => obj.Name.Equals(tableColumn.Group,StringComparison.OrdinalIgnoreCase))?.Columns ?? []; - table.AddColumn(new TableColumn(tableColumn.Group, table, tableColumn.Type, columns)); + table.AddColumn(new TableColumn(tableColumn.Name, tableColumn.Group, table, tableColumn.Type, columns)); } else { @@ -446,10 +446,11 @@ private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table, else if (xmlReader.NodeType == XmlNodeType.Element && xmlReader.Name.Equals("NavigationProperty", StringComparison.OrdinalIgnoreCase)) { //var containsTarget = xmlReader.GetAttribute("ContainsTarget"); - var columnTableTypeString = xmlReader.GetAttribute("Type"); - var columnTableType = GetColumnTableType(columnTableTypeString); - var tableName = GetTableName(columnTableTypeString); - tableColumn = new TableColumn(tableName, table, columnTableType, []); + var navigationPropertyName = xmlReader.GetAttribute("Name"); + var navigationPropertyTypeString = xmlReader.GetAttribute("Type"); + var navigationPropertyType = GetColumnTableType(navigationPropertyTypeString); + var groupName = GetTableName(navigationPropertyTypeString); + tableColumn = new TableColumn(navigationPropertyName, groupName, table, navigationPropertyType, []); table.AddTableColumn(tableColumn); } else if (xmlReader.Name.Equals("EntityType", StringComparison.OrdinalIgnoreCase) && xmlReader.GetAttribute("Name") != entityName) diff --git a/src/ODataSourceReader.cs b/src/ODataSourceReader.cs index 081ec86..0379b25 100644 --- a/src/ODataSourceReader.cs +++ b/src/ODataSourceReader.cs @@ -166,21 +166,30 @@ internal void CallEndpoing(IDictionary headers, bool readFromLas var selectAsParameters = GetSelectAsParameters(readFromLastRequestResponse); var modeAsParemters = GetModeAsParameters(); var filterAsParameters = GetFilterAsParameters(_mapping); + var expandAsParameters = GetExpandAsParameters(_mapping); + if (!string.IsNullOrEmpty(modeAsParemters)) { filterAsParameters.Add(modeAsParemters); } - if (selectAsParameters.Any()) + if (selectAsParameters.Count != 0) { parameters.Add("$select", string.Join(",", selectAsParameters)); } - if (filterAsParameters.Any()) + if (filterAsParameters.Count != 0) { parameters.Add("$filter", string.Join(" and ", filterAsParameters)); } + if (expandAsParameters.Count != 0) + { + //It is possible to use $select for the middle table by just separating select and expand with semicolon: + //Customers?$select=CustomerID&$expand=Orders($select=OrderID;$expand=Order_Details($select=UnitPrice)) + parameters.Add("$expand", string.Join(",", expandAsParameters)); + } + if (_endpoint.Parameters != null) { foreach (var parameter in _endpoint.Parameters) @@ -336,10 +345,10 @@ private string GetModeAsParameters() private List GetSelectAsParameters(bool readFromLastRequestResponse) { List result = new(); - var activeColumnMappings = _mapping.GetColumnMappings().Where(obj => obj.Active).ToList(); + var activeColumnMappings = _mapping.GetColumnMappings().Where(obj => obj.Active && obj.SourceColumn != null).ToList(); if (activeColumnMappings.Any()) { - var selectColumnNames = activeColumnMappings.Where(obj => obj.SourceColumn != null)?.Select(obj => obj.SourceColumn.Name).ToList(); + var selectColumnNames = activeColumnMappings.Where(obj => string.IsNullOrEmpty(obj.SourceColumn.Group))?.Select(obj => obj.SourceColumn.Name).ToList(); if (readFromLastRequestResponse) { @@ -507,6 +516,17 @@ private void LogWarningForConditional(MappingConditional item) _logger?.Warn($"Can only add {item.ConditionalOperator} on Edm.String and the {item.SourceColumn.Name} is a type of {item.SourceColumn.Type.Name} for the table mapping {_mapping.SourceTable.Name} to {_mapping.DestinationTable.Name}, so this have been removed from the $filter."); } + private List GetExpandAsParameters(Mapping mapping) + { + List result = new(); + var sourceColumnsWithGroups = mapping.SourceTable.Columns.Where(obj => !string.IsNullOrEmpty(obj.Group)); + if (sourceColumnsWithGroups.Any()) + { + result.AddRange(sourceColumnsWithGroups.DistinctBy(obj => obj.Group).Select(obj => obj.Name)); + } + return result; + } + /// /// Handles a specified response stream, by creating an IEnumerable with yield return, so we can later enumerate the result one object at a time without loading them all into memory /// From 6563a418245616f5fd85a369a115b666a1622cc2 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Wed, 29 Jan 2025 14:27:47 +0100 Subject: [PATCH 5/9] added replace for when working with default api on BC, still needs to figure out plurals for groups as that link is not working right now. added some logic for the writer, not done yet --- src/ODataProvider.cs | 1 + src/ODataWriter.cs | 68 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/src/ODataProvider.cs b/src/ODataProvider.cs index 63b4b7a..72437f5 100644 --- a/src/ODataProvider.cs +++ b/src/ODataProvider.cs @@ -515,6 +515,7 @@ private static Type GetColumnTableType(string columnTableTypeString) private static string GetTableName(string columnTableTypeString) { var result = columnTableTypeString.Replace("Collection(NAV.", "", StringComparison.OrdinalIgnoreCase); + result = result.Replace("Collection(Microsoft.NAV.", "", StringComparison.OrdinalIgnoreCase); return result.Replace(")", "", StringComparison.OrdinalIgnoreCase); } diff --git a/src/ODataWriter.cs b/src/ODataWriter.cs index fee0da7..5af284b 100644 --- a/src/ODataWriter.cs +++ b/src/ODataWriter.cs @@ -31,6 +31,7 @@ internal class ODataWriter : IDisposable, IDestinationWriter public Mapping Mapping { get; } internal JsonObject PostBackObject { get; set; } private readonly ColumnMappingCollection _columnMappings; + private readonly Dictionary _endpointExpand; internal ODataWriter(ILogger logger, Mapping mapping, Endpoint endpoint, ICredentials credentials, bool continueOnError) { @@ -45,12 +46,13 @@ internal ODataWriter(ILogger logger, Mapping mapping, Endpoint endpoint, ICreden _responseMappings = Mapping.GetResponseColumnMappings(); _continueOnError = continueOnError; _columnMappings = Mapping.GetColumnMappings(); + _endpointExpand = GetExpandForMapping(); } public void Write(Dictionary Row) { string endpointURL = Endpoint.Url; - string url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, ""); + string url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, "", _endpointExpand); var keyColumnValuesForFilter = GetKeyColumnValuesForFilter(Row); @@ -69,6 +71,16 @@ public void Write(Dictionary Row) } } } + if (_endpointExpand != null) + { + foreach (var item in _endpointExpand) + { + if (!parameters.ContainsKey(item.Key)) + { + parameters.Add(item.Key, item.Value); + } + } + } url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, "", parameters); var responseFromEndpoint = GetFromEndpoint(url, null); @@ -83,7 +95,7 @@ public void Write(Dictionary Row) var response = responseFromEndpoint?.Result?.Content?.Value; - url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, ""); + url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, "", _endpointExpand); if (response != null && response.Count > 0) { if (response.Count > 1) @@ -133,7 +145,7 @@ public void Write(Dictionary Row) if (primaryKeyColumnValuesForPatch.Any()) { string patchURL = "(" + string.Join(",", primaryKeyColumnValuesForPatch) + ")"; - url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, patchURL); + url = ODataSourceReader.GetEndpointURL(endpointURL, Mapping.DestinationTable.Name, patchURL, _endpointExpand); } awaitResponseFromEndpoint = PostToEndpoint(url, patchJson, headers, true); } @@ -294,7 +306,9 @@ internal string MapValuesToJson(Dictionary row, bool isPatchRequ { var jsonObject = new JsonObject(); - foreach (ColumnMapping columnMapping in _columnMappings) + var rootMappings = _columnMappings.Where(m => string.IsNullOrEmpty(m.DestinationColumn.Group)); + var nestedMappings = _columnMappings.Where(m => !string.IsNullOrEmpty(m.DestinationColumn.Group)); + foreach (ColumnMapping columnMapping in rootMappings) { if (!columnMapping.Active || (columnMapping.ScriptValueForInsert && isPatchRequest)) continue; @@ -326,6 +340,35 @@ internal string MapValuesToJson(Dictionary row, bool isPatchRequ } } } + if (nestedMappings.Any()) + { + var nestedJsonObject = new JsonObject(); + foreach (ColumnMapping cm in nestedMappings) + { + var columnValue = cm.ConvertInputValueToOutputValue(row.TryGetValue(cm.DestinationColumn.Group ?? "", out var value) ? value : null); + switch (cm.DestinationColumn.Type.Name.ToLower()) + { + case "decimal": + nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToDecimal(columnValue)); + break; + case "int": + nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToInt64(columnValue)); + break; + case "double": + nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToDecimal(columnValue)); + break; + case "datetime": + nestedJsonObject.Add(cm.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(columnValue, false)); + break; + case "dateonly": + nestedJsonObject.Add(cm.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(columnValue, true)); + break; + default: + nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToString(columnValue)); + break; + } + } + } return jsonObject.ToJsonString(); } @@ -357,6 +400,23 @@ public static string GetTheDateTimeInZeroTimeZone(object dateTimeObject, bool is return null; } + private Dictionary GetExpandForMapping() + { + if (Mapping.DestinationTable == null) + return []; + + var columnMappings = Mapping.GetColumnMappings(); + + if (!columnMappings.Any(column => !string.IsNullOrEmpty(column.DestinationColumn.Group))) + return []; + + var mappingGroups = columnMappings.DistinctBy(column => column.DestinationColumn.Group).Select(column => column.DestinationColumn.Name).ToList(); + if (mappingGroups.Count != 0) + return new Dictionary() { { "$expand", string.Join(",", mappingGroups) } }; + + return []; + } + public void Dispose() { } public void Close() { } From 2200dfa80955cdf7c7158c4038b8e14e020aa806 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Thu, 30 Jan 2025 15:10:40 +0100 Subject: [PATCH 6/9] changed how entitysets-tables are fetched so it is after all entitytypes-tables are created, and use table.sqlschema to store the entitytype-tablename to fetch columns for TableColumn. added extra .Replace so it now can handle default api for BC --- src/ODataProvider.cs | 47 +++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/ODataProvider.cs b/src/ODataProvider.cs index 72437f5..d96968c 100644 --- a/src/ODataProvider.cs +++ b/src/ODataProvider.cs @@ -360,9 +360,13 @@ void HandleStream(Stream responseStream, HttpStatusCode responseStatusCode, Dict else if (xmlReader.NodeType == XmlNodeType.Element && xmlReader.Name.Equals("EntitySet", StringComparison.OrdinalIgnoreCase)) { - GetColumnsFromEntityTypeTableToEntitySetTable(entitySetsTables.AddTable(xmlReader.GetAttribute("Name")), entityTypeTables, xmlReader.GetAttribute("EntityType")); + var entityTypeName = xmlReader.GetAttribute("EntityType"); + var SqlSchema = entityTypeName.Substring(entityTypeName.LastIndexOf(".") + 1); + var setTable = entitySetsTables.AddTable(xmlReader.GetAttribute("Name"), SqlSchema); } } + + GetColumnsFromEntityTypeTableToEntitySetTable(entityTypeTables, entitySetsTables); if (!EndpointIsLoadAllEntities(_endpoint.Url)) { var singleEntitySetSelected = entitySetsTables.GetTables().FirstOrDefault(obj => obj.Name.Equals(name, StringComparison.OrdinalIgnoreCase)); @@ -381,29 +385,39 @@ void HandleStream(Stream responseStream, HttpStatusCode responseStatusCode, Dict } } - private void GetColumnsFromEntityTypeTableToEntitySetTable(Table table, Schema entityTypeSchema, string entityTypeName) + private void GetColumnsFromEntityTypeTableToEntitySetTable(Schema entityTypeSchema, Schema entitySetsTables) { - var entityTypeNameClean = entityTypeName.Substring(entityTypeName.LastIndexOf(".") + 1); var entityTypeSchemaTables = entityTypeSchema.GetTables(); - Table result = entityTypeSchemaTables.Where(obj => obj.Name.Equals(entityTypeNameClean, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); - if (result != null) + var entitySetSchemaTables = entitySetsTables.GetTables(); + foreach (var table in entitySetSchemaTables) { - foreach (var item in result.Columns) + Table result = entityTypeSchemaTables.Where(obj => obj.Name.Equals(table.SqlSchema, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + if (result != null) { - if (table.Columns.Where(obj => obj.Name == item.Name).Count() == 0) + foreach (var item in result.Columns) { - if (item is TableColumn tableColumn) - { - var columns = entityTypeSchemaTables.FirstOrDefault(obj => obj.Name.Equals(tableColumn.Group,StringComparison.OrdinalIgnoreCase))?.Columns ?? []; - table.AddColumn(new TableColumn(tableColumn.Name, tableColumn.Group, table, tableColumn.Type, columns)); - } - else + if (table.Columns.Where(obj => obj.Name == item.Name).Count() == 0) { - table.AddColumn(new Column(item.Name, item.Type, table, item.IsPrimaryKey, item.IsNew, item.ReadOnly)); + if (item is TableColumn tableColumn) + { + var tableGroupName = tableColumn.Group; + var columns = entityTypeSchemaTables.FirstOrDefault(obj => obj.Name.Equals(tableColumn.Group, StringComparison.OrdinalIgnoreCase))?.Columns ?? []; + var entitySetTableName = entitySetSchemaTables.FirstOrDefault(obj => obj.SqlSchema.Equals(tableColumn.Group)); + if (entitySetTableName != null) + { + tableGroupName = entitySetTableName.Name; + } + table.AddColumn(new TableColumn(tableColumn.Name, tableGroupName, table, tableColumn.Type, columns)); + } + else + { + table.AddColumn(new Column(item.Name, item.Type, table, item.IsPrimaryKey, item.IsNew, item.ReadOnly)); + } } } } } + entitySetSchemaTables.ForEach(obj => obj.SqlSchema = string.Empty); } private void AddPropertiesFromXMLReaderToTable(XmlReader xmlReader, Table table, Schema result) @@ -514,8 +528,9 @@ private static Type GetColumnTableType(string columnTableTypeString) private static string GetTableName(string columnTableTypeString) { - var result = columnTableTypeString.Replace("Collection(NAV.", "", StringComparison.OrdinalIgnoreCase); - result = result.Replace("Collection(Microsoft.NAV.", "", StringComparison.OrdinalIgnoreCase); + var result = columnTableTypeString.Replace("Collection(", "", StringComparison.OrdinalIgnoreCase); + result = result.Replace("Microsoft.NAV.", "", StringComparison.OrdinalIgnoreCase); + result = result.Replace("NAV.", "", StringComparison.OrdinalIgnoreCase); return result.Replace(")", "", StringComparison.OrdinalIgnoreCase); } From f72428465bf4127605a794adf77157eac2639c15 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Mon, 3 Feb 2025 10:24:13 +0100 Subject: [PATCH 7/9] work so far --- src/ODataWriter.cs | 56 ++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/src/ODataWriter.cs b/src/ODataWriter.cs index 5af284b..a698776 100644 --- a/src/ODataWriter.cs +++ b/src/ODataWriter.cs @@ -342,31 +342,38 @@ internal string MapValuesToJson(Dictionary row, bool isPatchRequ } if (nestedMappings.Any()) { - var nestedJsonObject = new JsonObject(); - foreach (ColumnMapping cm in nestedMappings) + var nestedGroups = nestedMappings.DistinctBy(group => group.DestinationColumn.Group).Select(group => group.DestinationColumn.Group); + foreach (var group in nestedGroups) { - var columnValue = cm.ConvertInputValueToOutputValue(row.TryGetValue(cm.DestinationColumn.Group ?? "", out var value) ? value : null); - switch (cm.DestinationColumn.Type.Name.ToLower()) + var nestedJsonObject = new JsonObject(); + foreach (ColumnMapping cm in nestedMappings.Where(mapping => mapping.DestinationColumn.Group.Equals(group))) { - case "decimal": - nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToDecimal(columnValue)); - break; - case "int": - nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToInt64(columnValue)); - break; - case "double": - nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToDecimal(columnValue)); - break; - case "datetime": - nestedJsonObject.Add(cm.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(columnValue, false)); - break; - case "dateonly": - nestedJsonObject.Add(cm.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(columnValue, true)); - break; - default: - nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToString(columnValue)); - break; + row.TryGetValue(cm.SourceColumn?.Name ?? "", out var value); + + var columnValue = cm.ConvertInputValueToOutputValue(value); + switch (cm.DestinationColumn.Type.Name.ToLower()) + { + case "decimal": + nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToDecimal(columnValue)); + break; + case "int": + nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToInt64(columnValue)); + break; + case "double": + nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToDecimal(columnValue)); + break; + case "datetime": + nestedJsonObject.Add(cm.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(columnValue, false)); + break; + case "dateonly": + nestedJsonObject.Add(cm.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(columnValue, true)); + break; + default: + nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToString(columnValue)); + break; + } } + jsonObject.Add(group, nestedJsonObject); } } return jsonObject.ToJsonString(); @@ -407,10 +414,11 @@ private Dictionary GetExpandForMapping() var columnMappings = Mapping.GetColumnMappings(); - if (!columnMappings.Any(column => !string.IsNullOrEmpty(column.DestinationColumn.Group))) + var columnMappingGroups = columnMappings.Where(column => !string.IsNullOrEmpty(column.DestinationColumn.Group)); + if (!columnMappingGroups.Any()) return []; - var mappingGroups = columnMappings.DistinctBy(column => column.DestinationColumn.Group).Select(column => column.DestinationColumn.Name).ToList(); + var mappingGroups = columnMappingGroups.DistinctBy(column => column.DestinationColumn.Group).Select(column => column.DestinationColumn.Group).ToList(); if (mappingGroups.Count != 0) return new Dictionary() { { "$expand", string.Join(",", mappingGroups) } }; From 72159deeecefef7bed85075ba7a7a3cb0080859c Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Mon, 3 Feb 2025 16:05:23 +0100 Subject: [PATCH 8/9] work for today --- src/ODataWriter.cs | 65 ++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/src/ODataWriter.cs b/src/ODataWriter.cs index a698776..7bacf33 100644 --- a/src/ODataWriter.cs +++ b/src/ODataWriter.cs @@ -345,35 +345,50 @@ internal string MapValuesToJson(Dictionary row, bool isPatchRequ var nestedGroups = nestedMappings.DistinctBy(group => group.DestinationColumn.Group).Select(group => group.DestinationColumn.Group); foreach (var group in nestedGroups) { - var nestedJsonObject = new JsonObject(); - foreach (ColumnMapping cm in nestedMappings.Where(mapping => mapping.DestinationColumn.Group.Equals(group))) - { - row.TryGetValue(cm.SourceColumn?.Name ?? "", out var value); + var nestedMapping = nestedMappings.FirstOrDefault(mapping => mapping.DestinationColumn.Group.Equals(group)); + if (nestedMapping == null) + continue; - var columnValue = cm.ConvertInputValueToOutputValue(value); - switch (cm.DestinationColumn.Type.Name.ToLower()) + if (!row.TryGetValue(nestedMapping.SourceColumn.Group, out var nestedValue)) + { + continue; + } + if (nestedValue is List> nestedValueCollection) + { + var nestedJsonObjectList = new JsonArray(); + foreach (var nestedRow in nestedValueCollection) { - case "decimal": - nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToDecimal(columnValue)); - break; - case "int": - nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToInt64(columnValue)); - break; - case "double": - nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToDecimal(columnValue)); - break; - case "datetime": - nestedJsonObject.Add(cm.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(columnValue, false)); - break; - case "dateonly": - nestedJsonObject.Add(cm.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(columnValue, true)); - break; - default: - nestedJsonObject.Add(cm.DestinationColumn.Name, Converter.ToString(columnValue)); - break; + var nestedJsonObject = new JsonObject(); + foreach (ColumnMapping nestedColumnMapping in nestedMappings.Where(mapping => mapping.DestinationColumn.Group.Equals(group))) + { + var nestedColumnValue = nestedColumnMapping.ConvertInputValueToOutputValue(nestedColumnMapping.HasScriptWithValue ? null : nestedRow.TryGetValue(nestedColumnMapping.SourceColumn?.Name ?? "", out var value) ? value : null); + + switch (nestedColumnMapping.DestinationColumn.Type.Name.ToLower()) + { + case "decimal": + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToDecimal(nestedColumnValue)); + break; + case "int": + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToInt64(nestedColumnValue)); + break; + case "double": + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToDecimal(nestedColumnValue)); + break; + case "datetime": + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(nestedColumnValue, false)); + break; + case "dateonly": + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(nestedColumnValue, true)); + break; + default: + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToString(nestedColumnValue)); + break; + } + } + nestedJsonObjectList.Add(nestedJsonObject); } + jsonObject.Add(group, nestedJsonObjectList); } - jsonObject.Add(group, nestedJsonObject); } } return jsonObject.ToJsonString(); From ad68c4e92a9a07de158fb3dd90c079f7b15c8dc5 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Tue, 4 Feb 2025 12:33:52 +0100 Subject: [PATCH 9/9] work so far --- src/ODataWriter.cs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/ODataWriter.cs b/src/ODataWriter.cs index 7bacf33..9d6d2ca 100644 --- a/src/ODataWriter.cs +++ b/src/ODataWriter.cs @@ -334,12 +334,28 @@ internal string MapValuesToJson(Dictionary row, bool isPatchRequ case "dateonly": jsonObject.Add(columnMapping.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(columnValue, true)); break; + case "guid": + if (string.IsNullOrEmpty(Converter.ToString(columnValue))) + { + jsonObject.Add(columnMapping.DestinationColumn.Name, Converter.ToString(Guid.Empty)); + } + else + { + jsonObject.Add(columnMapping.DestinationColumn.Name, Converter.ToString(columnValue)); + } + break; default: jsonObject.Add(columnMapping.DestinationColumn.Name, Converter.ToString(columnValue)); break; } } } + //not able to patch/update nested objects + if (isPatchRequest) + { + return jsonObject.ToJsonString(); + } + if (nestedMappings.Any()) { var nestedGroups = nestedMappings.DistinctBy(group => group.DestinationColumn.Group).Select(group => group.DestinationColumn.Group); @@ -380,6 +396,16 @@ internal string MapValuesToJson(Dictionary row, bool isPatchRequ case "dateonly": nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, GetTheDateTimeInZeroTimeZone(nestedColumnValue, true)); break; + case "guid": + if (string.IsNullOrEmpty(Converter.ToString(nestedColumnValue))) + { + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToString(Guid.Empty)); + } + else + { + nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToString(nestedColumnValue)); + } + break; default: nestedJsonObject.Add(nestedColumnMapping.DestinationColumn.Name, Converter.ToString(nestedColumnValue)); break;