From 9e1c1c16ebd60f50d529d8366b096a8cf0b3c39b Mon Sep 17 00:00:00 2001 From: Frederik Nielsen Date: Wed, 24 Apr 2024 10:11:01 +0200 Subject: [PATCH 1/2] Creating a relations dictionary for tables and their foreign keys, then sort accordingly to that. We might be able to make a default implementation like this that takes a dictionary and orders things properly, but for now I have put it as a provider specific implementation to get some thoughts. --- ...aIntegration.Providers.EcomProvider.csproj | 8 +- src/EcomProvider.cs | 116 +++++------------- 2 files changed, 37 insertions(+), 87 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj index 19b0b4d..7211188 100644 --- a/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj @@ -1,6 +1,6 @@  - 10.0.24 + 10.0.25 1.0.0.0 Ecom Provider Ecom Provider @@ -14,7 +14,7 @@ Copyright © 2023 Dynamicweb Software A/S - net7.0 + net8.0 true true true @@ -23,8 +23,8 @@ snupkg - - + + diff --git a/src/EcomProvider.cs b/src/EcomProvider.cs index 47b1a8b..358a016 100644 --- a/src/EcomProvider.cs +++ b/src/EcomProvider.cs @@ -820,99 +820,49 @@ public override ISourceReader GetReader(Mapping mapping) return new EcomSourceReader(mapping, Connection, GetGroupNamesForVariantOptions, GetManufacturerNamesForProducts, GetGroupNamesForProduct, GetVariantGroupNamesForProduct, GetRelatedProductsByName, GetRelatedProductGroupsByName, SourceLanguage, SourceShop); } - public override void OrderTablesInJob(Job job, bool isSource) + private readonly Dictionary> tableRelations = new() { - MappingCollection tables = new MappingCollection(); - - var mappings = GetMappingsByName(job.Mappings, "EcomLanguages", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomCountries", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomCurrencies", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomStockLocation", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomGroups", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomManufacturers", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomVariantGroups", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomVariantsOptions", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomProducts", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomProductItems", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomProductsRelated", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomStockUnit", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomDetails", isSource); - if (mappings != null) - tables.AddRange(mappings); - - mappings = GetMappingsByName(job.Mappings, "EcomProductCategoryFieldValue", isSource); - if (mappings != null) - tables.AddRange(mappings); + { "EcomCurrencies", [ "EcomLanguages" ]}, + { "EcomGroups", [ "EcomLanguages" ]}, + { "EcomVariantGroups", [ "EcomLanguages" ]}, + { "EcomProducts", [ "EcomLanguages" ]}, + { "EcomAssortmentShopRelations", [ "EcomShops" ]} + }; + + public override bool IsSortable(Job job, bool isSource) + { + Func tableSelector = isSource ? + (map => map.SourceTable.Name) : + (map => map.DestinationTable.Name); - mappings = GetMappingsByName(job.Mappings, "EcomPrices", isSource); - if (mappings != null) - tables.AddRange(mappings); + var mappedTables = new HashSet(job.Mappings.Select(tableSelector)); - mappings = GetMappingsByName(job.Mappings, "EcomDiscount", isSource); - if (mappings != null) - tables.AddRange(mappings); + foreach (var (table, dependencies) in tableRelations) + if (mappedTables.Contains(table) && dependencies.All(mappedTables.Contains)) + return false; - mappings = GetMappingsByName(job.Mappings, "EcomAssortments", isSource); - if (mappings != null) - tables.AddRange(mappings); + return true; + } - mappings = GetMappingsByName(job.Mappings, "EcomAssortmentPermissions", isSource); - if (mappings != null) - tables.AddRange(mappings); + public override void OrderTablesInJob(Job job, bool isSource) + { + Func tableSelector = isSource ? + (map => map.SourceTable.Name) : + (map => map.DestinationTable.Name); - mappings = GetMappingsByName(job.Mappings, "EcomAssortmentGroupRelations", isSource); - if (mappings != null) - tables.AddRange(mappings); + HashSet priorityTables = tableRelations.Values.SelectMany(v => v).ToHashSet(); - mappings = GetMappingsByName(job.Mappings, "EcomAssortmentProductRelations", isSource); - if (mappings != null) - tables.AddRange(mappings); + if (!job.Mappings.Any(m => priorityTables.Any(relations => relations == tableSelector(m)))) + return; - mappings = GetMappingsByName(job.Mappings, "EcomAssortmentShopRelations", isSource); - if (mappings != null) - tables.AddRange(mappings); + var priorityMappings = job.Mappings + .Where(m => priorityTables.Contains(tableSelector(m))) + .OrderBy(m => priorityTables.IndexOf(tableSelector(m))); - mappings = GetMappingsByName(job.Mappings, "EcomVariantOptionsProductRelation", isSource); - if (mappings != null) - tables.AddRange(mappings); + var nonPriorityMappings = job.Mappings + .Where(m => !priorityTables.Contains(tableSelector(m))); - job.Mappings = tables; + job.Mappings = [.. priorityMappings.Concat(nonPriorityMappings)]; } internal static IEnumerable GetMappingsByName(MappingCollection collection, string name, bool isSource) From 008c4d02f80259cc1f13caace82fa63a250a1a36 Mon Sep 17 00:00:00 2001 From: Frederik Nielsen Date: Mon, 6 May 2024 08:27:30 +0200 Subject: [PATCH 2/2] Adding table relations to cover virtual columns --- src/EcomProvider.cs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/EcomProvider.cs b/src/EcomProvider.cs index 358a016..d2bb464 100644 --- a/src/EcomProvider.cs +++ b/src/EcomProvider.cs @@ -280,6 +280,18 @@ public EcomProvider(string connectionString) CreateMissingGoups = true; } + private readonly Dictionary> tableRelations = new() + { + { "EcomCurrencies", [ "EcomLanguages" ]}, + { "EcomGroups", [ "EcomLanguages", "EcomShops" ]}, + { "EcomVariantGroups", [ "EcomLanguages" ]}, + { "EcomProducts", [ "EcomLanguages", "EcomGroups", "EcomVariantGroups", "EcomVariantsOptions", "EcomProductsRelated" ]}, + { "EcomAssortmentShopRelations", [ "EcomShops" ]}, + { "EcomProductsRelated", [ "EcomLanguages" ] }, + { "EcomProductCategoryFieldValue", [ "EcomProducts" ] }, + { "EcomStockUnit", [ "EcomProducts", "EcomStockLocation" ] } + }; + public override Schema GetOriginalSourceSchema() { Schema result = GetDynamicwebSourceSchema(); @@ -820,15 +832,6 @@ public override ISourceReader GetReader(Mapping mapping) return new EcomSourceReader(mapping, Connection, GetGroupNamesForVariantOptions, GetManufacturerNamesForProducts, GetGroupNamesForProduct, GetVariantGroupNamesForProduct, GetRelatedProductsByName, GetRelatedProductGroupsByName, SourceLanguage, SourceShop); } - private readonly Dictionary> tableRelations = new() - { - { "EcomCurrencies", [ "EcomLanguages" ]}, - { "EcomGroups", [ "EcomLanguages" ]}, - { "EcomVariantGroups", [ "EcomLanguages" ]}, - { "EcomProducts", [ "EcomLanguages" ]}, - { "EcomAssortmentShopRelations", [ "EcomShops" ]} - }; - public override bool IsSortable(Job job, bool isSource) { Func tableSelector = isSource ? @@ -838,7 +841,7 @@ public override bool IsSortable(Job job, bool isSource) var mappedTables = new HashSet(job.Mappings.Select(tableSelector)); foreach (var (table, dependencies) in tableRelations) - if (mappedTables.Contains(table) && dependencies.All(mappedTables.Contains)) + if (mappedTables.Contains(table) && dependencies.Any(mappedTables.Contains)) return false; return true; @@ -850,7 +853,7 @@ public override void OrderTablesInJob(Job job, bool isSource) (map => map.SourceTable.Name) : (map => map.DestinationTable.Name); - HashSet priorityTables = tableRelations.Values.SelectMany(v => v).ToHashSet(); + var priorityTables = tableRelations.Values.SelectMany(v => v).Distinct().ToList(); if (!job.Mappings.Any(m => priorityTables.Any(relations => relations == tableSelector(m)))) return;