From 8860eb8c2e79a644b6e745968ce842ed8c18ffd7 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Mon, 30 Sep 2024 14:58:10 +0200 Subject: [PATCH 1/5] Remove missing rows - respect entire table, work so far, need a release of Dynamicweb.DataIntegration (10.9) so we reference that nuget instead of local .dll --- ...ration.Providers.DynamicwebProvider.csproj | 8 ++- src/DynamicwebBulkInsertDestinationWriter.cs | 12 +++++ src/DynamicwebProvider.cs | 49 +++++++++++++++++-- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj index 181563e..c9d6643 100644 --- a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj @@ -1,6 +1,6 @@  - 10.7.0 + 10.9.0 1.0.0.0 Dynamicweb Provider Dynamicweb Provider @@ -23,8 +23,12 @@ snupkg - + + + ..\..\..\Dynamicweb10\src\Features\DataIntegration\Dynamicweb.DataIntegration\bin\Debug\net8.0\Dynamicweb.DataIntegration.dll + + diff --git a/src/DynamicwebBulkInsertDestinationWriter.cs b/src/DynamicwebBulkInsertDestinationWriter.cs index e0964ad..83b6b09 100644 --- a/src/DynamicwebBulkInsertDestinationWriter.cs +++ b/src/DynamicwebBulkInsertDestinationWriter.cs @@ -30,6 +30,11 @@ public class DynamicwebBulkInsertDestinationWriter : BaseSqlWriter, IDestination protected readonly bool SkipFailingRows; public SqlCommand SqlCommand; + internal string GetTempTableName + { + get => tempTablePrefix; + } + public new Mapping Mapping { get; } private DataTable existingUsers; @@ -324,6 +329,13 @@ internal int DeleteExcessFromMainTable(string shop, SqlTransaction transaction, return result; } + internal int DeleteExcessFromMainTable(string shop, SqlTransaction transaction, Dictionary mappings) + { + SqlCommand.Transaction = transaction; + string extraConditions = GetExtraConditions(Mapping, shop, null); + return DeleteRowsFromMainTable(false, mappings, extraConditions, SqlCommand); + } + internal int DeleteExistingFromMainTable(string shop, SqlTransaction transaction) { SqlCommand.Transaction = transaction; diff --git a/src/DynamicwebProvider.cs b/src/DynamicwebProvider.cs index 5de05e7..9be261c 100644 --- a/src/DynamicwebProvider.cs +++ b/src/DynamicwebProvider.cs @@ -178,6 +178,11 @@ public string DefaultLanguage [AddInParameter("Remove missing rows after import"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows not present in the import source - including related tables. This option takes precedence. When Delete incoming rows is ON, this option is ignored"), AddInParameterGroup("Destination"), AddInParameterOrder(40)] public bool RemoveMissingAfterImport { get; set; } + [AddInParameter("Remove missing rows - respect entire table")] + [AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows not present in the import source")] + [AddInParameterGroup("Destination"), AddInParameterOrder(45)] + public bool RemoveMissingRows { get; set; } + [AddInParameter("Delete incoming rows"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes existing rows present in the import source. When Delete incoming rows is ON, the following options are skipped: Update only existing records, Deactivate missing products, Remove missing rows after import, Delete products / groups for languages included in input, Hide deactivated products"), AddInParameterGroup("Destination"), AddInParameterOrder(50)] public bool DeleteIncomingItems { get; set; } @@ -209,6 +214,7 @@ public string DefaultLanguage public override void SaveAsXml(XmlTextWriter xmlTextWriter) { + xmlTextWriter.WriteElementString("RemoveMissingRows", RemoveMissingRows.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("RemoveMissingAfterImport", RemoveMissingAfterImport.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("RemoveMissingAfterImportDestinationTablesOnly", RemoveMissingAfterImportDestinationTablesOnly.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("DeactivateMissingProducts", DeactivateMissingProducts.ToString(CultureInfo.CurrentCulture)); @@ -250,6 +256,12 @@ public DynamicwebProvider(XmlNode xmlNode) case "Schema": Schema = new Schema(node); break; + case "RemoveMissingRows": + if (node.HasChildNodes) + { + RemoveMissingRows = node.FirstChild.Value == "True"; + } + break; case "RemoveMissingAfterImport": if (node.HasChildNodes) { @@ -376,6 +388,7 @@ public override void UpdateDestinationSettings(IDestination destination) DiscardDuplicates = newProvider.DiscardDuplicates; HideDeactivatedProducts = newProvider.HideDeactivatedProducts; SkipFailingRows = newProvider.SkipFailingRows; + RemoveMissingRows = newProvider.RemoveMissingRows; } public override Schema GetOriginalSourceSchema() @@ -596,14 +609,39 @@ public override bool RunJob(Job job) } } - foreach (DynamicwebBulkInsertDestinationWriter writer in Enumerable.Reverse(Writers)) + if (RemoveMissingRows) { - bool? optionValue = writer.Mapping.GetOptionValue("DeleteIncomingItems"); - bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems; + var distinctWriters = Enumerable.Reverse(Writers).DistinctBy(obj => obj.Mapping.DestinationTable); + if (distinctWriters != null) + { + foreach (var distinctWriter in distinctWriters) + { + if (distinctWriter == null || distinctWriter.Mapping == null) + continue; + + var sameWriters = Writers.Where(obj => obj.Mapping != null && obj.Mapping.DestinationTable != null && obj.Mapping.DestinationTable.Name.Equals(distinctWriter.Mapping.DestinationTable?.Name ?? "", StringComparison.OrdinalIgnoreCase)).ToList(); + if (sameWriters.Count == 0) + continue; - if (!deleteIncomingItems && writer.RowsToWriteCount > 0) + Dictionary mappings = sameWriters.ToDictionary(obj => $"{obj.GetTempTableName}", obj => obj.Mapping); + if (mappings == null || mappings.Count == 0) + continue; + + TotalRowsAffected += sameWriters[0].DeleteExcessFromMainTable(Shop, sqlTransaction, mappings); + } + } + } + else + { + foreach (DynamicwebBulkInsertDestinationWriter writer in Enumerable.Reverse(Writers)) { - TotalRowsAffected += writer.DeleteExcessFromMainTable(Shop, sqlTransaction, DeleteProductsAndGroupForSpecificLanguage, defaultLanguage, HideDeactivatedProducts); + bool? optionValue = writer.Mapping.GetOptionValue("DeleteIncomingItems"); + bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems; + + if (!deleteIncomingItems && writer.RowsToWriteCount > 0) + { + TotalRowsAffected += writer.DeleteExcessFromMainTable(Shop, sqlTransaction, DeleteProductsAndGroupForSpecificLanguage, defaultLanguage, HideDeactivatedProducts); + } } } @@ -721,6 +759,7 @@ public override string Serialize() root.Add(CreateParameterNode(GetType(), "Deactivate missing products", DeactivateMissingProducts.ToString())); root.Add(CreateParameterNode(GetType(), "Update only existing records", UpdateOnlyExistingRecords.ToString())); root.Add(CreateParameterNode(GetType(), "Insert only new records", InsertOnlyNewRecords.ToString())); + root.Add(CreateParameterNode(GetType(), "Remove missing rows - respect entire table", RemoveMissingRows.ToString())); root.Add(CreateParameterNode(GetType(), "Remove missing rows after import", RemoveMissingAfterImport.ToString())); root.Add(CreateParameterNode(GetType(), "Remove missing rows after import in the destination tables only", RemoveMissingAfterImportDestinationTablesOnly.ToString())); root.Add(CreateParameterNode(GetType(), "Delete incoming rows", DeleteIncomingItems.ToString())); From fbc192fe2ee55f28f077e9b56f7811909ba105a6 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Thu, 19 Dec 2024 13:10:35 +0100 Subject: [PATCH 2/5] added tooltil for both new addin and updated the old-addin --- ...eb.DataIntegration.Providers.DynamicwebProvider.csproj | 8 ++------ src/DynamicwebProvider.cs | 6 +++--- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj index c9d6643..adfc843 100644 --- a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj @@ -23,12 +23,8 @@ snupkg - + + - - - ..\..\..\Dynamicweb10\src\Features\DataIntegration\Dynamicweb.DataIntegration\bin\Debug\net8.0\Dynamicweb.DataIntegration.dll - - diff --git a/src/DynamicwebProvider.cs b/src/DynamicwebProvider.cs index 9be261c..e891a7e 100644 --- a/src/DynamicwebProvider.cs +++ b/src/DynamicwebProvider.cs @@ -175,11 +175,11 @@ public string DefaultLanguage [AddInParameter("Remove missing rows after import in the destination tables only"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows not present in the import source - excluding related tabled"), AddInParameterGroup("Destination"), AddInParameterOrder(35)] public bool RemoveMissingAfterImportDestinationTablesOnly { get; set; } - [AddInParameter("Remove missing rows after import"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows not present in the import source - including related tables. This option takes precedence. When Delete incoming rows is ON, this option is ignored"), AddInParameterGroup("Destination"), AddInParameterOrder(40)] + [AddInParameter("Remove missing rows after import"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows from each destination table individually, based on whether they are present in the corresponding source table. This setting looks at each table separately and removes rows missing from the source for that specific table. When Delete incoming rows is ON, this option is ignored"), AddInParameterGroup("Destination"), AddInParameterOrder(40)] public bool RemoveMissingAfterImport { get; set; } - [AddInParameter("Remove missing rows - respect entire table")] - [AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows not present in the import source")] + [AddInParameter("Remove missing rows across all tables after import")] + [AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows from all destination tables and relation tables by considering the entire dataset in the import source. This setting evaluates all tables collectively and removes rows missing across the whole activity.")] [AddInParameterGroup("Destination"), AddInParameterOrder(45)] public bool RemoveMissingRows { get; set; } From 26c325fc4687a0dd911a01e5c9e8a4e7df12d3fb Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Thu, 19 Dec 2024 13:12:09 +0100 Subject: [PATCH 3/5] updated Serialize() --- src/DynamicwebProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DynamicwebProvider.cs b/src/DynamicwebProvider.cs index e891a7e..c372cf9 100644 --- a/src/DynamicwebProvider.cs +++ b/src/DynamicwebProvider.cs @@ -759,7 +759,7 @@ public override string Serialize() root.Add(CreateParameterNode(GetType(), "Deactivate missing products", DeactivateMissingProducts.ToString())); root.Add(CreateParameterNode(GetType(), "Update only existing records", UpdateOnlyExistingRecords.ToString())); root.Add(CreateParameterNode(GetType(), "Insert only new records", InsertOnlyNewRecords.ToString())); - root.Add(CreateParameterNode(GetType(), "Remove missing rows - respect entire table", RemoveMissingRows.ToString())); + root.Add(CreateParameterNode(GetType(), "Remove missing rows across all tables after import", RemoveMissingRows.ToString())); root.Add(CreateParameterNode(GetType(), "Remove missing rows after import", RemoveMissingAfterImport.ToString())); root.Add(CreateParameterNode(GetType(), "Remove missing rows after import in the destination tables only", RemoveMissingAfterImportDestinationTablesOnly.ToString())); root.Add(CreateParameterNode(GetType(), "Delete incoming rows", DeleteIncomingItems.ToString())); From 04d8eb18a123d52cfa622b3b1024446a675f5d65 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Thu, 19 Dec 2024 13:14:50 +0100 Subject: [PATCH 4/5] fixing inlines in .csproj --- ...micweb.DataIntegration.Providers.DynamicwebProvider.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj index adfc843..a474471 100644 --- a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj @@ -24,7 +24,7 @@ - - + + From 43a1f511252cd95e4c23360c6dcdd26b8af615ab Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Thu, 19 Dec 2024 13:15:19 +0100 Subject: [PATCH 5/5] inlines again --- ...ration.Providers.DynamicwebProvider.csproj | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj index a474471..4ae2c2c 100644 --- a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj @@ -1,30 +1,30 @@  - + 10.9.0 - 1.0.0.0 - Dynamicweb Provider - Dynamicweb Provider - - - https://doc.dynamicweb.com/ - Dynamicweb dw10 addin integration providers - Dynamicweb CMS - Dynamicweb Software A/S - Dynamicweb Software A/S - Copyright © 2023 Dynamicweb Software A/S - - - net8.0 - true - true - true - true - true - snupkg - - + 1.0.0.0 + Dynamicweb Provider + Dynamicweb Provider + + + https://doc.dynamicweb.com/ + Dynamicweb dw10 addin integration providers + Dynamicweb CMS + Dynamicweb Software A/S + Dynamicweb Software A/S + Copyright © 2023 Dynamicweb Software A/S + + + net8.0 + true + true + true + true + true + snupkg + + - +