From 90dd5f63c2d6785dd10d4d6adf8cff6262bd3b71 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Thu, 9 Jan 2025 09:36:51 +0100 Subject: [PATCH 1/3] Added new addin for remove-missing-rows. added logic in destinationwriter to handle new addin for removing missing rows. --- ...aIntegration.Providers.EcomProvider.csproj | 6 +- src/EcomDestinationWriter.cs | 72 ++++++++++--------- src/EcomProvider.cs | 48 +++++++++++-- 3 files changed, 85 insertions(+), 41 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj index bda430d..648b10a 100644 --- a/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj @@ -1,6 +1,6 @@  - 10.8.3 + 10.9.0 1.0.0.0 Ecom Provider Ecom Provider @@ -24,8 +24,8 @@ enable - - + + diff --git a/src/EcomDestinationWriter.cs b/src/EcomDestinationWriter.cs index 72f4b38..485408f 100644 --- a/src/EcomDestinationWriter.cs +++ b/src/EcomDestinationWriter.cs @@ -57,6 +57,7 @@ internal class EcomDestinationWriter : BaseSqlWriter protected internal Dictionary>> DataRowsToWrite = new Dictionary>>(StringComparer.OrdinalIgnoreCase); private Dictionary ImportedProductsByNumber = new Dictionary(StringComparer.OrdinalIgnoreCase); private List _addedMappingsForMoveToMainTables = new List(); + internal static string GetTempTableName => "TempTableForBulkImport"; internal int RowsToWriteCount { @@ -315,7 +316,7 @@ internal void CreateTempTables() { foreach (var mapping in tableMappings) { - CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport" + mapping.GetId(), destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, GetTempTableName + mapping.GetId(), destColumns, logger); AddTableToDataset(destColumns, GetTableName(table.Name, mapping)); } } @@ -330,42 +331,42 @@ internal void CreateTempTables() break; case "EcomGroups": EnsureDestinationColumns(currentTable, null, destColumns, ["GroupID", "GroupLanguageID", "GroupName"]); - CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomVariantGroups": EnsureDestinationColumns(currentTable, null, destColumns, ["VariantGroupID", "VariantGroupLanguageID", "VariantGroupName"]); - CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomVariantsOptions": EnsureDestinationColumns(currentTable, null, destColumns, ["VariantOptionID", "VariantOptionLanguageID", "VariantOptionName"]); - CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomManufacturers": EnsureDestinationColumns(currentTable, null, destColumns, ["ManufacturerID", "ManufacturerName"]); - CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomProductsRelated": EnsureDestinationColumns(currentTable, null, destColumns, ["ProductRelatedProductID", "ProductRelatedProductRelID", "ProductRelatedGroupID", "ProductRelatedProductRelVariantID"]); - CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomLanguages": EnsureDestinationColumns(currentTable, null, destColumns, ["LanguageID", "LanguageCode2", "LanguageName", "LanguageNativeName"]); - CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomVariantOptionsProductRelation": EnsureDestinationColumns(currentTable, null, destColumns, ["VariantOptionsProductRelationProductID", "VariantOptionsProductRelationVariantID"]); - CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomProductCategoryFieldValue": EnsureDestinationColumns(currentTable, null, destColumns, ["FieldValueFieldId", "FieldValueFieldCategoryId", "FieldValueProductId", "FieldValueProductVariantId", "FieldValueProductLanguageId", "FieldValueValue"]); - CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; } @@ -376,42 +377,42 @@ internal void CreateTempTables() var relationTable = GetTable("EcomGroupProductRelation", currentTables); var groupProductRelationColumns = new List(); EnsureDestinationColumns(relationTable, null, groupProductRelationColumns, ["GroupProductRelationGroupId", "GroupProductRelationProductId", "GroupProductRelationSorting", "GroupProductRelationIsPrimary"]); - CreateTempTable(null, "EcomGroupProductRelation", "TempTableForBulkImport", groupProductRelationColumns, logger); + CreateTempTable(null, "EcomGroupProductRelation", GetTempTableName, groupProductRelationColumns, logger); AddTableToDataset(groupProductRelationColumns, "EcomGroupProductRelation"); //create product variantgroup relation temp table List variantGroupProductRelation = new List(); relationTable = GetTable("EcomVariantgroupProductRelation", currentTables); EnsureDestinationColumns(relationTable, null, variantGroupProductRelation, ["VariantgroupProductRelationProductID", "VariantgroupProductRelationVariantGroupID", "VariantgroupProductRelationID", "VariantGroupProductRelationSorting"]); - CreateTempTable(null, "EcomVariantgroupProductRelation", "TempTableForBulkImport", variantGroupProductRelation, logger); + CreateTempTable(null, "EcomVariantgroupProductRelation", GetTempTableName, variantGroupProductRelation, logger); AddTableToDataset(variantGroupProductRelation, "EcomVariantgroupProductRelation"); //Create ShopGroupRelation temp table List shopGroupRelations = new List(); relationTable = GetTable("EcomShopGroupRelation", currentTables); EnsureDestinationColumns(relationTable, null, shopGroupRelations, ["ShopGroupShopID", "ShopGroupGroupID", "ShopGroupRelationsSorting"]); - CreateTempTable(null, "EcomShopGroupRelation", "TempTableForBulkImport", shopGroupRelations, logger); + CreateTempTable(null, "EcomShopGroupRelation", GetTempTableName, shopGroupRelations, logger); AddTableToDataset(shopGroupRelations, "EcomShopGroupRelation"); //Create Shop relation table List shops = new List(); relationTable = GetTable("EcomShops", currentTables); EnsureDestinationColumns(relationTable, null, shops, ["ShopID", "ShopName"]); - CreateTempTable(null, "EcomShops", "TempTableForBulkImport", shops, logger); + CreateTempTable(null, "EcomShops", GetTempTableName, shops, logger); AddTableToDataset(shops, "EcomShops"); //Create Product-relatedGroup temp table List productsRelatedGroups = new List(); relationTable = GetTable("EcomProductsRelatedGroups", currentTables); EnsureDestinationColumns(relationTable, null, productsRelatedGroups, ["RelatedGroupID", "RelatedGroupName", "RelatedGroupLanguageID"]); - CreateTempTable(null, "EcomProductsRelatedGroups", "TempTableForBulkImport", productsRelatedGroups, logger); + CreateTempTable(null, "EcomProductsRelatedGroups", GetTempTableName, productsRelatedGroups, logger); AddTableToDataset(productsRelatedGroups, "EcomProductsRelatedGroups"); //Create EcomGroupRelations temp table List groupRelations = new List(); relationTable = GetTable("EcomGroupRelations", currentTables); EnsureDestinationColumns(relationTable, null, groupRelations, ["GroupRelationsGroupID", "GroupRelationsParentID", "GroupRelationsSorting"]); - CreateTempTable(null, "EcomGroupRelations", "TempTableForBulkImport", groupRelations, logger); + CreateTempTable(null, "EcomGroupRelations", GetTempTableName, groupRelations, logger); AddTableToDataset(groupRelations, "EcomGroupRelations"); } @@ -441,14 +442,6 @@ private static void EnsureDestinationColumns(Table table, Dictionary columnMappingDictionary, List destColumns, Dictionary destinationTableColumns, string columnName) - { - if (!columnMappingDictionary.ContainsKey(columnName)) - { - destColumns.Add((SqlColumn)destinationTableColumns[columnName]); - } } private string GetTableName(string name, Mapping mapping) @@ -3003,7 +2996,7 @@ public void FinishWriting() } using (SqlBulkCopy sqlBulkCopier = new SqlBulkCopy(connection)) { - sqlBulkCopier.DestinationTableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName); + sqlBulkCopier.DestinationTableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); sqlBulkCopier.BulkCopyTimeout = 0; int skippedFailedRowsCount = 0; try @@ -3032,11 +3025,22 @@ public void FinishWriting() } } + public int DeleteExcessFromMainTable(string shop, SqlTransaction transaction, Dictionary mappings) + { + sqlCommand.Transaction = transaction; + var mapping = mappings.Values.FirstOrDefault(); + if (mapping is null) + return 0; + + string extraConditions = GetExtraConditions(mapping, shop, null); + return DeleteRowsFromMainTable(false, mappings, extraConditions, sqlCommand); + } + public void DeleteExcessFromMainTable(string shop, SqlTransaction transaction, string languageId, bool deleteProductsAndGroupForSpecificLanguage, bool hideDeactivatedProducts) { foreach (Mapping mapping in job.Mappings.Where(m => !_addedMappingsForMoveToMainTables.Contains(m))) { - string tempTablePrefix = "TempTableForBulkImport" + mapping.GetId(); + string tempTablePrefix = GetTempTableName + mapping.GetId(); if (HasRowsToImport(mapping, out tempTablePrefix)) { if ((mapping.DestinationTable.Name == "EcomProducts" || mapping.DestinationTable.Name == "EcomGroups") && deleteProductsAndGroupForSpecificLanguage) @@ -3086,7 +3090,7 @@ public void DeleteExistingFromMainTable(string shop, SqlTransaction transaction, sqlCommand.Transaction = transaction; foreach (Mapping mapping in job.Mappings) { - string tempTablePrefix = "TempTableForBulkImport" + mapping.GetId(); + string tempTablePrefix = GetTempTableName + mapping.GetId(); if (HasRowsToImport(mapping, out tempTablePrefix)) { var rowsAffected = DeleteExistingFromMainTable(sqlCommand, mapping, GetExtraConditions(mapping, shop, languageId), tempTablePrefix); @@ -3154,7 +3158,7 @@ public void MoveDataToMainTables(string shop, SqlTransaction sqlTransaction, boo var mappingId = mapping.GetId(); if (mapping.Active && ColumnMappingsByMappingId[mappingId].Count > 0) { - string tempTablePrefix = "TempTableForBulkImport" + mappingId; + string tempTablePrefix = GetTempTableName + mappingId; if (HasRowsToImport(mapping, out tempTablePrefix)) { bool? optionValue = mapping.GetOptionValue("UpdateOnlyExistingRecords"); @@ -4133,7 +4137,7 @@ private Dictionary> GetEcomProductsPKColumns() { foreach (DataTable table in DataToWrite.Tables) { - string tableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName); + string tableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); sqlCommand.CommandText = $"if exists (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'{tableName}') AND type in (N'U')) drop table [{tableName}]"; sqlCommand.ExecuteNonQuery(); } @@ -4193,7 +4197,7 @@ private void RemoveExcessFromRelationsTables(SqlTransaction sqlTransaction) foreach (DataTable table in FindDataTablesStartingWithName("EcomVariantOptionsProductRelation")) { - string tempTableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName); + string tempTableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); if (deleteExcess || removeMissingAfterImportDestinationTablesOnly) { sqlCommand.CommandText += $"delete from EcomVariantOptionsProductRelation where VariantOptionsProductRelationProductID in (select VariantOptionsProductRelationProductID from {tempTableName}); "; @@ -4208,7 +4212,7 @@ private void RemoveExcessFromRelationsTables(SqlTransaction sqlTransaction) foreach (DataTable table in FindDataTablesStartingWithName("EcomShops")) { - string tempTableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName); + string tempTableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); sqlCommand.CommandText += $"insert into EcomShops (ShopID,ShopName) select shopid,shopname from {tempTableName}; "; } @@ -4216,7 +4220,7 @@ private void RemoveExcessFromRelationsTables(SqlTransaction sqlTransaction) foreach (DataTable table in FindDataTablesStartingWithName("EcomProductsRelated")) { - string tempTableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName); + string tempTableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); sqlCommand.CommandText += "delete from related from EcomProductsRelated related where ProductRelatedProductID in " + $"(select ProductRelatedProductID from {tempTableName} inside WHERE related.ProductRelatedProductID = inside.ProductRelatedProductID AND " + "related.ProductRelatedProductRelID = inside.ProductRelatedProductRelID AND related.ProductRelatedGroupID = inside.ProductRelatedGroupID AND related.ProductRelatedProductRelVariantID = inside.ProductRelatedProductRelVariantID); "; @@ -4252,7 +4256,7 @@ private void DeleteExcessFromGroupProductRelation(string shop, SqlTransaction sq sqlClean = new StringBuilder(); foreach (DataTable table in FindDataTablesStartingWithName("EcomProducts")) { - string tempTableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName); + string tempTableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); sqlClean.Append($"delete EcomGroupProductRelation from {tempTableName} join ecomgroupproductrelation on {tempTableName}.productid=ecomgroupproductrelation.GroupProductRelationProductID where not exists (select * from [dbo].[EcomGroupProductRelationTempTableForBulkImport] where [dbo].[EcomGroupProductRelation].[GroupProductRelationProductID]=[GroupProductRelationProductID] and [dbo].[EcomGroupProductRelation].[GroupProductRelationGroupID]=[GroupProductRelationGroupID] );"); } } @@ -4290,7 +4294,7 @@ private void DeleteExcessFromGroupProductRelation(string shop, SqlTransaction sq private bool HasRowsToImport(Mapping? mapping, out string tempTablePrefix) { bool result = false; - tempTablePrefix = "TempTableForBulkImport" + mapping?.GetId(); + tempTablePrefix = GetTempTableName + mapping?.GetId(); if (mapping != null && mapping.DestinationTable != null && mapping.DestinationTable.Name != null && DataToWrite != null && DataToWrite.Tables != null) { @@ -4302,7 +4306,7 @@ private bool HasRowsToImport(Mapping? mapping, out string tempTablePrefix) } else if (DataRowsToWrite.TryGetValue(mapping.DestinationTable.Name, out rows) && rows.Values.Count > 0) { - tempTablePrefix = "TempTableForBulkImport"; + tempTablePrefix = GetTempTableName; result = true; } } diff --git a/src/EcomProvider.cs b/src/EcomProvider.cs index b69afa7..7f3a225 100644 --- a/src/EcomProvider.cs +++ b/src/EcomProvider.cs @@ -180,21 +180,27 @@ public string DefaultLanguage public bool UseStrictPrimaryKeyMatching { get; set; } [AddInParameter("Remove missing rows after import")] - [AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Removes rows from the destination and relation tables. This option takes precedence")] + [AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows from each destination table individually, based on whether they are present in the corresponding source table. This setting looks at each table separately and removes rows missing from the source for that specific table")] [AddInParameterGroup("Destination")] [AddInParameterOrder(40)] public bool RemoveMissingAfterImport { get; set; } + [AddInParameter("Remove missing rows across all tables after import")] + [AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows from all destination tables and relation tables by considering the entire dataset in the import source. This setting evaluates all tables collectively and removes rows missing across the whole activity.")] + [AddInParameterGroup("Destination")] + [AddInParameterOrder(41)] + public bool RemoveMissingRows { get; set; } + [AddInParameter("Update only existing products")] [AddInParameterEditor(typeof(YesNoParameterEditor), "")] [AddInParameterGroup("Destination")] - [AddInParameterOrder(41)] + [AddInParameterOrder(42)] public bool UpdateOnlyExistingProducts { get; set; } [AddInParameter("Create missing groups")] [AddInParameterEditor(typeof(YesNoParameterEditor), "")] [AddInParameterGroup("Destination")] - [AddInParameterOrder(42)] + [AddInParameterOrder(43)] public bool CreateMissingGoups { get; set; } [AddInParameter("Delete incoming rows")] @@ -499,6 +505,12 @@ public EcomProvider(XmlNode xmlNode) case "Schema": Schema = new Schema(node); break; + case "RemoveMissingRows": + if (node.HasChildNodes) + { + RemoveMissingRows = node.FirstChild?.Value == "True"; + } + break; case "RemoveMissingAfterImport": if (node.HasChildNodes) { @@ -698,6 +710,7 @@ void ISource.SaveAsXml(XmlTextWriter xmlTextWriter) void IDestination.SaveAsXml(XmlTextWriter xmlTextWriter) { xmlTextWriter.WriteElementString("RemoveMissingAfterImport", RemoveMissingAfterImport.ToString(CultureInfo.CurrentCulture)); + xmlTextWriter.WriteElementString("RemoveMissingRows", RemoveMissingRows.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("RemoveMissingAfterImportDestinationTablesOnly", RemoveMissingAfterImportDestinationTablesOnly.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("DeactivateMissingProducts", DeactivateMissingProducts.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("DeleteProductsAndGroupForSpecificLanguage", DeleteProductsAndGroupForSpecificLanguage.ToString(CultureInfo.CurrentCulture)); @@ -753,6 +766,7 @@ public override void UpdateDestinationSettings(IDestination destination) IgnoreEmptyCategoryFieldValues = newProvider.IgnoreEmptyCategoryFieldValues; RemoveMissingAfterImport = newProvider.RemoveMissingAfterImport; RemoveMissingAfterImportDestinationTablesOnly = newProvider.RemoveMissingAfterImportDestinationTablesOnly; + RemoveMissingRows = newProvider.RemoveMissingRows; } public override string Serialize() @@ -775,6 +789,7 @@ public override string Serialize() root.Add(CreateParameterNode(GetType(), "User key field", UserKeyField ?? "")); root.Add(CreateParameterNode(GetType(), "Remove missing rows after import", RemoveMissingAfterImport.ToString())); root.Add(CreateParameterNode(GetType(), "Remove missing rows after import in the destination tables only", RemoveMissingAfterImportDestinationTablesOnly.ToString())); + root.Add(CreateParameterNode(GetType(), "Remove missing rows across all tables after import", RemoveMissingRows.ToString())); root.Add(CreateParameterNode(GetType(), "Update only existing products", UpdateOnlyExistingProducts.ToString())); root.Add(CreateParameterNode(GetType(), "Use strict primary key matching", UseStrictPrimaryKeyMatching.ToString())); root.Add(CreateParameterNode(GetType(), "Update only existing records", UpdateOnlyExistingRecords.ToString())); @@ -1008,7 +1023,32 @@ public override bool RunJob(Job job) else { Writer.MoveDataToMainTables(Shop, sqlTransaction, UpdateOnlyExistingRecords, InsertOnlyNewRecords); - Writer.DeleteExcessFromMainTable(Shop, sqlTransaction, DefaultLanguage, DeleteProductsAndGroupForSpecificLanguage, HideDeactivatedProducts); + if (RemoveMissingRows) + { + var distinctWriters = job.Mappings.DistinctBy(obj => obj.DestinationTable); + if (distinctWriters != null) + { + foreach (var distinctWriter in distinctWriters) + { + if (distinctWriter == null) + continue; + + var sameWriters = job.Mappings.Where(obj => obj != null && obj.DestinationTable != null && obj.DestinationTable.Name.Equals(distinctWriter.DestinationTable?.Name ?? "", StringComparison.OrdinalIgnoreCase)).ToList(); + if (sameWriters.Count == 0) + continue; + + Dictionary mappings = sameWriters.ToDictionary(obj => $"{EcomDestinationWriter.GetTempTableName}{obj.GetId()}", obj => obj); + if (mappings == null || mappings.Count == 0) + continue; + + TotalRowsAffected += Writer.DeleteExcessFromMainTable(Shop, sqlTransaction, mappings); + } + } + } + else + { + Writer.DeleteExcessFromMainTable(Shop, sqlTransaction, DefaultLanguage, DeleteProductsAndGroupForSpecificLanguage, HideDeactivatedProducts); + } } Writer.CleanRelationsTables(sqlTransaction); sqlTransaction.Commit(); From 812c17f5a4b53a82c64ff5f94ed18dac6d8213a0 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Thu, 16 Jan 2025 14:20:51 +0100 Subject: [PATCH 2/3] code-review feedback --- src/EcomDestinationWriter.cs | 54 +++++++++++++++---------------- src/EcomProvider.cs | 62 ++++++++++++++++++++---------------- 2 files changed, 62 insertions(+), 54 deletions(-) diff --git a/src/EcomDestinationWriter.cs b/src/EcomDestinationWriter.cs index 485408f..efd5e75 100644 --- a/src/EcomDestinationWriter.cs +++ b/src/EcomDestinationWriter.cs @@ -57,7 +57,7 @@ internal class EcomDestinationWriter : BaseSqlWriter protected internal Dictionary>> DataRowsToWrite = new Dictionary>>(StringComparer.OrdinalIgnoreCase); private Dictionary ImportedProductsByNumber = new Dictionary(StringComparer.OrdinalIgnoreCase); private List _addedMappingsForMoveToMainTables = new List(); - internal static string GetTempTableName => "TempTableForBulkImport"; + internal const string TempTableName = "TempTableForBulkImport"; internal int RowsToWriteCount { @@ -316,7 +316,7 @@ internal void CreateTempTables() { foreach (var mapping in tableMappings) { - CreateTempTable(table.SqlSchema, table.Name, GetTempTableName + mapping.GetId(), destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, TempTableName + mapping.GetId(), destColumns, logger); AddTableToDataset(destColumns, GetTableName(table.Name, mapping)); } } @@ -331,42 +331,42 @@ internal void CreateTempTables() break; case "EcomGroups": EnsureDestinationColumns(currentTable, null, destColumns, ["GroupID", "GroupLanguageID", "GroupName"]); - CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomVariantGroups": EnsureDestinationColumns(currentTable, null, destColumns, ["VariantGroupID", "VariantGroupLanguageID", "VariantGroupName"]); - CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomVariantsOptions": EnsureDestinationColumns(currentTable, null, destColumns, ["VariantOptionID", "VariantOptionLanguageID", "VariantOptionName"]); - CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomManufacturers": EnsureDestinationColumns(currentTable, null, destColumns, ["ManufacturerID", "ManufacturerName"]); - CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomProductsRelated": EnsureDestinationColumns(currentTable, null, destColumns, ["ProductRelatedProductID", "ProductRelatedProductRelID", "ProductRelatedGroupID", "ProductRelatedProductRelVariantID"]); - CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomLanguages": EnsureDestinationColumns(currentTable, null, destColumns, ["LanguageID", "LanguageCode2", "LanguageName", "LanguageNativeName"]); - CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomVariantOptionsProductRelation": EnsureDestinationColumns(currentTable, null, destColumns, ["VariantOptionsProductRelationProductID", "VariantOptionsProductRelationVariantID"]); - CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; case "EcomProductCategoryFieldValue": EnsureDestinationColumns(currentTable, null, destColumns, ["FieldValueFieldId", "FieldValueFieldCategoryId", "FieldValueProductId", "FieldValueProductVariantId", "FieldValueProductLanguageId", "FieldValueValue"]); - CreateTempTable(table.SqlSchema, table.Name, GetTempTableName, destColumns, logger); + CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger); AddTableToDataset(destColumns, table.Name); break; } @@ -377,42 +377,42 @@ internal void CreateTempTables() var relationTable = GetTable("EcomGroupProductRelation", currentTables); var groupProductRelationColumns = new List(); EnsureDestinationColumns(relationTable, null, groupProductRelationColumns, ["GroupProductRelationGroupId", "GroupProductRelationProductId", "GroupProductRelationSorting", "GroupProductRelationIsPrimary"]); - CreateTempTable(null, "EcomGroupProductRelation", GetTempTableName, groupProductRelationColumns, logger); + CreateTempTable(null, "EcomGroupProductRelation", TempTableName, groupProductRelationColumns, logger); AddTableToDataset(groupProductRelationColumns, "EcomGroupProductRelation"); //create product variantgroup relation temp table List variantGroupProductRelation = new List(); relationTable = GetTable("EcomVariantgroupProductRelation", currentTables); EnsureDestinationColumns(relationTable, null, variantGroupProductRelation, ["VariantgroupProductRelationProductID", "VariantgroupProductRelationVariantGroupID", "VariantgroupProductRelationID", "VariantGroupProductRelationSorting"]); - CreateTempTable(null, "EcomVariantgroupProductRelation", GetTempTableName, variantGroupProductRelation, logger); + CreateTempTable(null, "EcomVariantgroupProductRelation", TempTableName, variantGroupProductRelation, logger); AddTableToDataset(variantGroupProductRelation, "EcomVariantgroupProductRelation"); //Create ShopGroupRelation temp table List shopGroupRelations = new List(); relationTable = GetTable("EcomShopGroupRelation", currentTables); EnsureDestinationColumns(relationTable, null, shopGroupRelations, ["ShopGroupShopID", "ShopGroupGroupID", "ShopGroupRelationsSorting"]); - CreateTempTable(null, "EcomShopGroupRelation", GetTempTableName, shopGroupRelations, logger); + CreateTempTable(null, "EcomShopGroupRelation", TempTableName, shopGroupRelations, logger); AddTableToDataset(shopGroupRelations, "EcomShopGroupRelation"); //Create Shop relation table List shops = new List(); relationTable = GetTable("EcomShops", currentTables); EnsureDestinationColumns(relationTable, null, shops, ["ShopID", "ShopName"]); - CreateTempTable(null, "EcomShops", GetTempTableName, shops, logger); + CreateTempTable(null, "EcomShops", TempTableName, shops, logger); AddTableToDataset(shops, "EcomShops"); //Create Product-relatedGroup temp table List productsRelatedGroups = new List(); relationTable = GetTable("EcomProductsRelatedGroups", currentTables); EnsureDestinationColumns(relationTable, null, productsRelatedGroups, ["RelatedGroupID", "RelatedGroupName", "RelatedGroupLanguageID"]); - CreateTempTable(null, "EcomProductsRelatedGroups", GetTempTableName, productsRelatedGroups, logger); + CreateTempTable(null, "EcomProductsRelatedGroups", TempTableName, productsRelatedGroups, logger); AddTableToDataset(productsRelatedGroups, "EcomProductsRelatedGroups"); //Create EcomGroupRelations temp table List groupRelations = new List(); relationTable = GetTable("EcomGroupRelations", currentTables); EnsureDestinationColumns(relationTable, null, groupRelations, ["GroupRelationsGroupID", "GroupRelationsParentID", "GroupRelationsSorting"]); - CreateTempTable(null, "EcomGroupRelations", GetTempTableName, groupRelations, logger); + CreateTempTable(null, "EcomGroupRelations", TempTableName, groupRelations, logger); AddTableToDataset(groupRelations, "EcomGroupRelations"); } @@ -2996,7 +2996,7 @@ public void FinishWriting() } using (SqlBulkCopy sqlBulkCopier = new SqlBulkCopy(connection)) { - sqlBulkCopier.DestinationTableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); + sqlBulkCopier.DestinationTableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName); sqlBulkCopier.BulkCopyTimeout = 0; int skippedFailedRowsCount = 0; try @@ -3040,7 +3040,7 @@ public void DeleteExcessFromMainTable(string shop, SqlTransaction transaction, s { foreach (Mapping mapping in job.Mappings.Where(m => !_addedMappingsForMoveToMainTables.Contains(m))) { - string tempTablePrefix = GetTempTableName + mapping.GetId(); + string tempTablePrefix = TempTableName + mapping.GetId(); if (HasRowsToImport(mapping, out tempTablePrefix)) { if ((mapping.DestinationTable.Name == "EcomProducts" || mapping.DestinationTable.Name == "EcomGroups") && deleteProductsAndGroupForSpecificLanguage) @@ -3090,7 +3090,7 @@ public void DeleteExistingFromMainTable(string shop, SqlTransaction transaction, sqlCommand.Transaction = transaction; foreach (Mapping mapping in job.Mappings) { - string tempTablePrefix = GetTempTableName + mapping.GetId(); + string tempTablePrefix = TempTableName + mapping.GetId(); if (HasRowsToImport(mapping, out tempTablePrefix)) { var rowsAffected = DeleteExistingFromMainTable(sqlCommand, mapping, GetExtraConditions(mapping, shop, languageId), tempTablePrefix); @@ -3158,7 +3158,7 @@ public void MoveDataToMainTables(string shop, SqlTransaction sqlTransaction, boo var mappingId = mapping.GetId(); if (mapping.Active && ColumnMappingsByMappingId[mappingId].Count > 0) { - string tempTablePrefix = GetTempTableName + mappingId; + string tempTablePrefix = TempTableName + mappingId; if (HasRowsToImport(mapping, out tempTablePrefix)) { bool? optionValue = mapping.GetOptionValue("UpdateOnlyExistingRecords"); @@ -4137,7 +4137,7 @@ private Dictionary> GetEcomProductsPKColumns() { foreach (DataTable table in DataToWrite.Tables) { - string tableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); + string tableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName); sqlCommand.CommandText = $"if exists (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'{tableName}') AND type in (N'U')) drop table [{tableName}]"; sqlCommand.ExecuteNonQuery(); } @@ -4197,7 +4197,7 @@ private void RemoveExcessFromRelationsTables(SqlTransaction sqlTransaction) foreach (DataTable table in FindDataTablesStartingWithName("EcomVariantOptionsProductRelation")) { - string tempTableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); + string tempTableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName); if (deleteExcess || removeMissingAfterImportDestinationTablesOnly) { sqlCommand.CommandText += $"delete from EcomVariantOptionsProductRelation where VariantOptionsProductRelationProductID in (select VariantOptionsProductRelationProductID from {tempTableName}); "; @@ -4212,7 +4212,7 @@ private void RemoveExcessFromRelationsTables(SqlTransaction sqlTransaction) foreach (DataTable table in FindDataTablesStartingWithName("EcomShops")) { - string tempTableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); + string tempTableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName); sqlCommand.CommandText += $"insert into EcomShops (ShopID,ShopName) select shopid,shopname from {tempTableName}; "; } @@ -4220,7 +4220,7 @@ private void RemoveExcessFromRelationsTables(SqlTransaction sqlTransaction) foreach (DataTable table in FindDataTablesStartingWithName("EcomProductsRelated")) { - string tempTableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); + string tempTableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName); sqlCommand.CommandText += "delete from related from EcomProductsRelated related where ProductRelatedProductID in " + $"(select ProductRelatedProductID from {tempTableName} inside WHERE related.ProductRelatedProductID = inside.ProductRelatedProductID AND " + "related.ProductRelatedProductRelID = inside.ProductRelatedProductRelID AND related.ProductRelatedGroupID = inside.ProductRelatedGroupID AND related.ProductRelatedProductRelVariantID = inside.ProductRelatedProductRelVariantID); "; @@ -4256,7 +4256,7 @@ private void DeleteExcessFromGroupProductRelation(string shop, SqlTransaction sq sqlClean = new StringBuilder(); foreach (DataTable table in FindDataTablesStartingWithName("EcomProducts")) { - string tempTableName = GetTableNameWithoutPrefix(table.TableName) + GetTempTableName + GetPrefixFromTableName(table.TableName); + string tempTableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName); sqlClean.Append($"delete EcomGroupProductRelation from {tempTableName} join ecomgroupproductrelation on {tempTableName}.productid=ecomgroupproductrelation.GroupProductRelationProductID where not exists (select * from [dbo].[EcomGroupProductRelationTempTableForBulkImport] where [dbo].[EcomGroupProductRelation].[GroupProductRelationProductID]=[GroupProductRelationProductID] and [dbo].[EcomGroupProductRelation].[GroupProductRelationGroupID]=[GroupProductRelationGroupID] );"); } } @@ -4294,7 +4294,7 @@ private void DeleteExcessFromGroupProductRelation(string shop, SqlTransaction sq private bool HasRowsToImport(Mapping? mapping, out string tempTablePrefix) { bool result = false; - tempTablePrefix = GetTempTableName + mapping?.GetId(); + tempTablePrefix = TempTableName + mapping?.GetId(); if (mapping != null && mapping.DestinationTable != null && mapping.DestinationTable.Name != null && DataToWrite != null && DataToWrite.Tables != null) { @@ -4306,7 +4306,7 @@ private bool HasRowsToImport(Mapping? mapping, out string tempTablePrefix) } else if (DataRowsToWrite.TryGetValue(mapping.DestinationTable.Name, out rows) && rows.Values.Count > 0) { - tempTablePrefix = GetTempTableName; + tempTablePrefix = TempTableName; result = true; } } diff --git a/src/EcomProvider.cs b/src/EcomProvider.cs index 7f3a225..8a2f7a1 100644 --- a/src/EcomProvider.cs +++ b/src/EcomProvider.cs @@ -286,11 +286,11 @@ public EcomProvider(string connectionString) CreateMissingGoups = true; } - string ISource.GetId() => "Source|EcomProvider"; + string ISource.GetId() => "Source|EcomProvider"; - string IDestination.GetId() => "Destination|EcomProvider"; + string IDestination.GetId() => "Destination|EcomProvider"; - public override Schema GetOriginalDestinationSchema() + public override Schema GetOriginalDestinationSchema() { Schema result = GetOriginalSourceSchema(); foreach (Table table in result.GetTables()) @@ -691,7 +691,7 @@ public override string ValidateSourceSettings() return ""; } - void ISource.SaveAsXml(XmlTextWriter xmlTextWriter) + void ISource.SaveAsXml(XmlTextWriter xmlTextWriter) { xmlTextWriter.WriteElementString("SqlConnectionString", SqlConnectionString); xmlTextWriter.WriteElementString("SourceShop", SourceShop); @@ -710,7 +710,7 @@ void ISource.SaveAsXml(XmlTextWriter xmlTextWriter) void IDestination.SaveAsXml(XmlTextWriter xmlTextWriter) { xmlTextWriter.WriteElementString("RemoveMissingAfterImport", RemoveMissingAfterImport.ToString(CultureInfo.CurrentCulture)); - xmlTextWriter.WriteElementString("RemoveMissingRows", RemoveMissingRows.ToString(CultureInfo.CurrentCulture)); + xmlTextWriter.WriteElementString("RemoveMissingRows", RemoveMissingRows.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("RemoveMissingAfterImportDestinationTablesOnly", RemoveMissingAfterImportDestinationTablesOnly.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("DeactivateMissingProducts", DeactivateMissingProducts.ToString(CultureInfo.CurrentCulture)); xmlTextWriter.WriteElementString("DeleteProductsAndGroupForSpecificLanguage", DeleteProductsAndGroupForSpecificLanguage.ToString(CultureInfo.CurrentCulture)); @@ -728,8 +728,8 @@ void IDestination.SaveAsXml(XmlTextWriter xmlTextWriter) xmlTextWriter.WriteElementString(nameof(UseProductIdFoundByNumber), UseProductIdFoundByNumber.ToString()); xmlTextWriter.WriteElementString(nameof(IgnoreEmptyCategoryFieldValues), IgnoreEmptyCategoryFieldValues.ToString()); xmlTextWriter.WriteElementString("SkipFailingRows", SkipFailingRows.ToString(CultureInfo.CurrentCulture)); - if (!Feature.IsActive()) - (this as IDestination).GetSchema().SaveAsXml(xmlTextWriter); + if (!Feature.IsActive()) + (this as IDestination).GetSchema().SaveAsXml(xmlTextWriter); } public override void UpdateSourceSettings(ISource source) @@ -1025,25 +1025,7 @@ public override bool RunJob(Job job) Writer.MoveDataToMainTables(Shop, sqlTransaction, UpdateOnlyExistingRecords, InsertOnlyNewRecords); if (RemoveMissingRows) { - var distinctWriters = job.Mappings.DistinctBy(obj => obj.DestinationTable); - if (distinctWriters != null) - { - foreach (var distinctWriter in distinctWriters) - { - if (distinctWriter == null) - continue; - - var sameWriters = job.Mappings.Where(obj => obj != null && obj.DestinationTable != null && obj.DestinationTable.Name.Equals(distinctWriter.DestinationTable?.Name ?? "", StringComparison.OrdinalIgnoreCase)).ToList(); - if (sameWriters.Count == 0) - continue; - - Dictionary mappings = sameWriters.ToDictionary(obj => $"{EcomDestinationWriter.GetTempTableName}{obj.GetId()}", obj => obj); - if (mappings == null || mappings.Count == 0) - continue; - - TotalRowsAffected += Writer.DeleteExcessFromMainTable(Shop, sqlTransaction, mappings); - } - } + RemoveMissingRowsAcrossAllTables(job, sqlTransaction); } else { @@ -1055,7 +1037,7 @@ public override bool RunJob(Job job) Writer.RebuildAssortments(); TotalRowsAffected += Writer.RowsAffected; - + MoveRepositoriesIndexToJob(job); } catch (Exception ex) @@ -1105,6 +1087,32 @@ public override bool RunJob(Job job) return true; } + private void RemoveMissingRowsAcrossAllTables(Job job, SqlTransaction? sqlTransaction) + { + if (sqlTransaction is null) + return; + + var distinctWriters = job.Mappings.DistinctBy(obj => obj.DestinationTable); + if (distinctWriters is null) + return; + + foreach (var distinctWriter in distinctWriters) + { + if (distinctWriter == null) + continue; + + var sameWriters = job.Mappings.Where(obj => obj != null && obj.DestinationTable != null && obj.DestinationTable.Name.Equals(distinctWriter.DestinationTable?.Name ?? "", StringComparison.OrdinalIgnoreCase)).ToList(); + if (sameWriters.Count == 0) + continue; + + Dictionary mappings = sameWriters.ToDictionary(obj => $"{EcomDestinationWriter.TempTableName}{obj.GetId()}", obj => obj); + if (mappings == null || mappings.Count == 0) + continue; + + TotalRowsAffected += Writer?.DeleteExcessFromMainTable(Shop, sqlTransaction, mappings) ?? 0; + } + } + private void MoveRepositoriesIndexToJob(Job job) { if (!string.IsNullOrEmpty(RepositoriesIndexUpdate)) From 9d85f94b070d54e65e7a1d19edbb6520bbf64e29 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Tue, 11 Feb 2025 13:08:47 +0100 Subject: [PATCH 3/3] bump version --- src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj index 648b10a..200b03a 100644 --- a/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj @@ -1,6 +1,6 @@  - 10.9.0 + 10.9.1 1.0.0.0 Ecom Provider Ecom Provider