diff --git a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj index cdb1e03..4ae2c2c 100644 --- a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj @@ -1,30 +1,30 @@  - - 10.8.1 - 1.0.0.0 - Dynamicweb Provider - Dynamicweb Provider - - - https://doc.dynamicweb.com/ - Dynamicweb dw10 addin integration providers - Dynamicweb CMS - Dynamicweb Software A/S - Dynamicweb Software A/S - Copyright © 2023 Dynamicweb Software A/S - - - net8.0 - true - true - true - true - true - snupkg - - - - - - + + 10.9.0 + 1.0.0.0 + Dynamicweb Provider + Dynamicweb Provider + + + https://doc.dynamicweb.com/ + Dynamicweb dw10 addin integration providers + Dynamicweb CMS + Dynamicweb Software A/S + Dynamicweb Software A/S + Copyright © 2023 Dynamicweb Software A/S + + + net8.0 + true + true + true + true + true + snupkg + + + + + + diff --git a/src/DynamicwebBulkInsertDestinationWriter.cs b/src/DynamicwebBulkInsertDestinationWriter.cs index e0964ad..83b6b09 100644 --- a/src/DynamicwebBulkInsertDestinationWriter.cs +++ b/src/DynamicwebBulkInsertDestinationWriter.cs @@ -30,6 +30,11 @@ public class DynamicwebBulkInsertDestinationWriter : BaseSqlWriter, IDestination protected readonly bool SkipFailingRows; public SqlCommand SqlCommand; + internal string GetTempTableName + { + get => tempTablePrefix; + } + public new Mapping Mapping { get; } private DataTable existingUsers; @@ -324,6 +329,13 @@ internal int DeleteExcessFromMainTable(string shop, SqlTransaction transaction, return result; } + internal int DeleteExcessFromMainTable(string shop, SqlTransaction transaction, Dictionary mappings) + { + SqlCommand.Transaction = transaction; + string extraConditions = GetExtraConditions(Mapping, shop, null); + return DeleteRowsFromMainTable(false, mappings, extraConditions, SqlCommand); + } + internal int DeleteExistingFromMainTable(string shop, SqlTransaction transaction) { SqlCommand.Transaction = transaction; diff --git a/src/DynamicwebProvider.cs b/src/DynamicwebProvider.cs index 32fcfce..19b5728 100644 --- a/src/DynamicwebProvider.cs +++ b/src/DynamicwebProvider.cs @@ -176,9 +176,14 @@ public string DefaultLanguage [AddInParameter("Remove missing rows after import in the destination tables only"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows not present in the import source - excluding related tabled"), AddInParameterGroup("Destination"), AddInParameterOrder(35)] public bool RemoveMissingAfterImportDestinationTablesOnly { get; set; } - [AddInParameter("Remove missing rows after import"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows not present in the import source - including related tables. This option takes precedence. When Delete incoming rows is ON, this option is ignored"), AddInParameterGroup("Destination"), AddInParameterOrder(40)] + [AddInParameter("Remove missing rows after import"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows from each destination table individually, based on whether they are present in the corresponding source table. This setting looks at each table separately and removes rows missing from the source for that specific table. When Delete incoming rows is ON, this option is ignored"), AddInParameterGroup("Destination"), AddInParameterOrder(40)] public bool RemoveMissingAfterImport { get; set; } + [AddInParameter("Remove missing rows across all tables after import")] + [AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows from all destination tables and relation tables by considering the entire dataset in the import source. This setting evaluates all tables collectively and removes rows missing across the whole activity.")] + [AddInParameterGroup("Destination"), AddInParameterOrder(45)] + public bool RemoveMissingRows { get; set; } + [AddInParameter("Delete incoming rows"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes existing rows present in the import source. When Delete incoming rows is ON, the following options are skipped: Update only existing records, Deactivate missing products, Remove missing rows after import, Delete products / groups for languages included in input, Hide deactivated products"), AddInParameterGroup("Destination"), AddInParameterOrder(50)] public bool DeleteIncomingItems { get; set; } @@ -210,6 +215,7 @@ public string DefaultLanguage public override void SaveAsXml(XmlTextWriter xmlTextWriter) { + xmlTextWriter.WriteElementString("RemoveMissingRows", RemoveMissingRows.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("RemoveMissingAfterImport", RemoveMissingAfterImport.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("RemoveMissingAfterImportDestinationTablesOnly", RemoveMissingAfterImportDestinationTablesOnly.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("DeactivateMissingProducts", DeactivateMissingProducts.ToString(CultureInfo.CurrentCulture)); @@ -257,6 +263,12 @@ public DynamicwebProvider(XmlNode xmlNode) case "Schema": Schema = new Schema(node); break; + case "RemoveMissingRows": + if (node.HasChildNodes) + { + RemoveMissingRows = node.FirstChild.Value == "True"; + } + break; case "RemoveMissingAfterImport": if (node.HasChildNodes) { @@ -383,6 +395,7 @@ public override void UpdateDestinationSettings(IDestination destination) DiscardDuplicates = newProvider.DiscardDuplicates; HideDeactivatedProducts = newProvider.HideDeactivatedProducts; SkipFailingRows = newProvider.SkipFailingRows; + RemoveMissingRows = newProvider.RemoveMissingRows; } public override Schema GetOriginalSourceSchema() @@ -585,14 +598,39 @@ public override bool RunJob(Job job) } } - foreach (DynamicwebBulkInsertDestinationWriter writer in Enumerable.Reverse(Writers)) + if (RemoveMissingRows) { - bool? optionValue = writer.Mapping.GetOptionValue("DeleteIncomingItems"); - bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems; + var distinctWriters = Enumerable.Reverse(Writers).DistinctBy(obj => obj.Mapping.DestinationTable); + if (distinctWriters != null) + { + foreach (var distinctWriter in distinctWriters) + { + if (distinctWriter == null || distinctWriter.Mapping == null) + continue; + + var sameWriters = Writers.Where(obj => obj.Mapping != null && obj.Mapping.DestinationTable != null && obj.Mapping.DestinationTable.Name.Equals(distinctWriter.Mapping.DestinationTable?.Name ?? "", StringComparison.OrdinalIgnoreCase)).ToList(); + if (sameWriters.Count == 0) + continue; - if (!deleteIncomingItems && writer.RowsToWriteCount > 0) + Dictionary mappings = sameWriters.ToDictionary(obj => $"{obj.GetTempTableName}", obj => obj.Mapping); + if (mappings == null || mappings.Count == 0) + continue; + + TotalRowsAffected += sameWriters[0].DeleteExcessFromMainTable(Shop, sqlTransaction, mappings); + } + } + } + else + { + foreach (DynamicwebBulkInsertDestinationWriter writer in Enumerable.Reverse(Writers)) { - TotalRowsAffected += writer.DeleteExcessFromMainTable(Shop, sqlTransaction, DeleteProductsAndGroupForSpecificLanguage, defaultLanguage, HideDeactivatedProducts); + bool? optionValue = writer.Mapping.GetOptionValue("DeleteIncomingItems"); + bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems; + + if (!deleteIncomingItems && writer.RowsToWriteCount > 0) + { + TotalRowsAffected += writer.DeleteExcessFromMainTable(Shop, sqlTransaction, DeleteProductsAndGroupForSpecificLanguage, defaultLanguage, HideDeactivatedProducts); + } } } @@ -710,6 +748,7 @@ public override string Serialize() root.Add(CreateParameterNode(GetType(), "Deactivate missing products", DeactivateMissingProducts.ToString())); root.Add(CreateParameterNode(GetType(), "Update only existing records", UpdateOnlyExistingRecords.ToString())); root.Add(CreateParameterNode(GetType(), "Insert only new records", InsertOnlyNewRecords.ToString())); + root.Add(CreateParameterNode(GetType(), "Remove missing rows across all tables after import", RemoveMissingRows.ToString())); root.Add(CreateParameterNode(GetType(), "Remove missing rows after import", RemoveMissingAfterImport.ToString())); root.Add(CreateParameterNode(GetType(), "Remove missing rows after import in the destination tables only", RemoveMissingAfterImportDestinationTablesOnly.ToString())); root.Add(CreateParameterNode(GetType(), "Delete incoming rows", DeleteIncomingItems.ToString()));