Skip to content

Added new addin for remove-missing-rows. added logic in destinationwr… #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
frederik5480 merged 4 commits into from
Mar 7, 2025
Merged

Added new addin for remove-missing-rows. added logic in destinationwr… #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
90dd5f6
Added new addin for remove-missing-rows. added logic in destinationwr…
MatthiasSort Jan 9, 2025
812c17f
code-review feedback
MatthiasSort Jan 16, 2025
2526672
Merged master into branch to handle Conflicts
MatthiasSort Feb 11, 2025
9d85f94
bump version
MatthiasSort Feb 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/Dynamicweb.DataIntegration.Providers.EcomProvider.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Version>10.8.4</Version>
<Version>10.9.1</Version>
<AssemblyVersion>1.0.0.0</AssemblyVersion>
<Title>Ecom Provider</Title>
<Description>Ecom Provider</Description>
Expand All @@ -24,8 +24,8 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dynamicweb.DataIntegration" Version="10.8.0" />
<PackageReference Include="Dynamicweb.Ecommerce" Version="10.8.0" />
<PackageReference Include="Dynamicweb.DataIntegration" Version="10.9.0" />
<PackageReference Include="Dynamicweb.Ecommerce" Version="10.9.0" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
</ItemGroup>
</Project>
72 changes: 38 additions & 34 deletions src/EcomDestinationWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ internal class EcomDestinationWriter : BaseSqlWriter
protected internal Dictionary<string, Dictionary<string, List<DataRow>>> DataRowsToWrite = new Dictionary<string, Dictionary<string, List<DataRow>>>(StringComparer.OrdinalIgnoreCase);
private Dictionary<string, DataRow> ImportedProductsByNumber = new Dictionary<string, DataRow>(StringComparer.OrdinalIgnoreCase);
private List<Mapping> _addedMappingsForMoveToMainTables = new List<Mapping>();
internal const string TempTableName = "TempTableForBulkImport";

internal int RowsToWriteCount
{
Expand Down Expand Up @@ -315,7 +316,7 @@ internal void CreateTempTables()
{
foreach (var mapping in tableMappings)
{
CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport" + mapping.GetId(), destColumns, logger);
CreateTempTable(table.SqlSchema, table.Name, TempTableName + mapping.GetId(), destColumns, logger);
AddTableToDataset(destColumns, GetTableName(table.Name, mapping));
}
}
Expand All @@ -330,42 +331,42 @@ internal void CreateTempTables()
break;
case "EcomGroups":
EnsureDestinationColumns(currentTable, null, destColumns, ["GroupID", "GroupLanguageID", "GroupName"]);
CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger);
CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger);
AddTableToDataset(destColumns, table.Name);
break;
case "EcomVariantGroups":
EnsureDestinationColumns(currentTable, null, destColumns, ["VariantGroupID", "VariantGroupLanguageID", "VariantGroupName"]);
CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger);
CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger);
AddTableToDataset(destColumns, table.Name);
break;
case "EcomVariantsOptions":
EnsureDestinationColumns(currentTable, null, destColumns, ["VariantOptionID", "VariantOptionLanguageID", "VariantOptionName"]);
CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger);
CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger);
AddTableToDataset(destColumns, table.Name);
break;
case "EcomManufacturers":
EnsureDestinationColumns(currentTable, null, destColumns, ["ManufacturerID", "ManufacturerName"]);
CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger);
CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger);
AddTableToDataset(destColumns, table.Name);
break;
case "EcomProductsRelated":
EnsureDestinationColumns(currentTable, null, destColumns, ["ProductRelatedProductID", "ProductRelatedProductRelID", "ProductRelatedGroupID", "ProductRelatedProductRelVariantID"]);
CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger);
CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger);
AddTableToDataset(destColumns, table.Name);
break;
case "EcomLanguages":
EnsureDestinationColumns(currentTable, null, destColumns, ["LanguageID", "LanguageCode2", "LanguageName", "LanguageNativeName"]);
CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger);
CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger);
AddTableToDataset(destColumns, table.Name);
break;
case "EcomVariantOptionsProductRelation":
EnsureDestinationColumns(currentTable, null, destColumns, ["VariantOptionsProductRelationProductID", "VariantOptionsProductRelationVariantID"]);
CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger);
CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger);
AddTableToDataset(destColumns, table.Name);
break;
case "EcomProductCategoryFieldValue":
EnsureDestinationColumns(currentTable, null, destColumns, ["FieldValueFieldId", "FieldValueFieldCategoryId", "FieldValueProductId", "FieldValueProductVariantId", "FieldValueProductLanguageId", "FieldValueValue"]);
CreateTempTable(table.SqlSchema, table.Name, "TempTableForBulkImport", destColumns, logger);
CreateTempTable(table.SqlSchema, table.Name, TempTableName, destColumns, logger);
AddTableToDataset(destColumns, table.Name);
break;
}
Expand All @@ -376,42 +377,42 @@ internal void CreateTempTables()
var relationTable = GetTable("EcomGroupProductRelation", currentTables);
var groupProductRelationColumns = new List<SqlColumn>();
EnsureDestinationColumns(relationTable, null, groupProductRelationColumns, ["GroupProductRelationGroupId", "GroupProductRelationProductId", "GroupProductRelationSorting", "GroupProductRelationIsPrimary"]);
CreateTempTable(null, "EcomGroupProductRelation", "TempTableForBulkImport", groupProductRelationColumns, logger);
CreateTempTable(null, "EcomGroupProductRelation", TempTableName, groupProductRelationColumns, logger);
AddTableToDataset(groupProductRelationColumns, "EcomGroupProductRelation");

//create product variantgroup relation temp table
List<SqlColumn> variantGroupProductRelation = new List<SqlColumn>();
relationTable = GetTable("EcomVariantgroupProductRelation", currentTables);
EnsureDestinationColumns(relationTable, null, variantGroupProductRelation, ["VariantgroupProductRelationProductID", "VariantgroupProductRelationVariantGroupID", "VariantgroupProductRelationID", "VariantGroupProductRelationSorting"]);
CreateTempTable(null, "EcomVariantgroupProductRelation", "TempTableForBulkImport", variantGroupProductRelation, logger);
CreateTempTable(null, "EcomVariantgroupProductRelation", TempTableName, variantGroupProductRelation, logger);
AddTableToDataset(variantGroupProductRelation, "EcomVariantgroupProductRelation");

//Create ShopGroupRelation temp table
List<SqlColumn> shopGroupRelations = new List<SqlColumn>();
relationTable = GetTable("EcomShopGroupRelation", currentTables);
EnsureDestinationColumns(relationTable, null, shopGroupRelations, ["ShopGroupShopID", "ShopGroupGroupID", "ShopGroupRelationsSorting"]);
CreateTempTable(null, "EcomShopGroupRelation", "TempTableForBulkImport", shopGroupRelations, logger);
CreateTempTable(null, "EcomShopGroupRelation", TempTableName, shopGroupRelations, logger);
AddTableToDataset(shopGroupRelations, "EcomShopGroupRelation");

//Create Shop relation table
List<SqlColumn> shops = new List<SqlColumn>();
relationTable = GetTable("EcomShops", currentTables);
EnsureDestinationColumns(relationTable, null, shops, ["ShopID", "ShopName"]);
CreateTempTable(null, "EcomShops", "TempTableForBulkImport", shops, logger);
CreateTempTable(null, "EcomShops", TempTableName, shops, logger);
AddTableToDataset(shops, "EcomShops");

//Create Product-relatedGroup temp table
List<SqlColumn> productsRelatedGroups = new List<SqlColumn>();
relationTable = GetTable("EcomProductsRelatedGroups", currentTables);
EnsureDestinationColumns(relationTable, null, productsRelatedGroups, ["RelatedGroupID", "RelatedGroupName", "RelatedGroupLanguageID"]);
CreateTempTable(null, "EcomProductsRelatedGroups", "TempTableForBulkImport", productsRelatedGroups, logger);
CreateTempTable(null, "EcomProductsRelatedGroups", TempTableName, productsRelatedGroups, logger);
AddTableToDataset(productsRelatedGroups, "EcomProductsRelatedGroups");

//Create EcomGroupRelations temp table
List<SqlColumn> groupRelations = new List<SqlColumn>();
relationTable = GetTable("EcomGroupRelations", currentTables);
EnsureDestinationColumns(relationTable, null, groupRelations, ["GroupRelationsGroupID", "GroupRelationsParentID", "GroupRelationsSorting"]);
CreateTempTable(null, "EcomGroupRelations", "TempTableForBulkImport", groupRelations, logger);
CreateTempTable(null, "EcomGroupRelations", TempTableName, groupRelations, logger);
AddTableToDataset(groupRelations, "EcomGroupRelations");
}

Expand Down Expand Up @@ -441,14 +442,6 @@ private static void EnsureDestinationColumns(Table table, Dictionary<string, Col
destColumns.Add(sqlColumn);
}
}
}

private static void EnsureDestinationColumn(Dictionary<string, Column> columnMappingDictionary, List<SqlColumn> destColumns, Dictionary<string, Column> destinationTableColumns, string columnName)
{
if (!columnMappingDictionary.ContainsKey(columnName))
{
destColumns.Add((SqlColumn)destinationTableColumns[columnName]);
}
}

private string GetTableName(string name, Mapping mapping)
Expand Down Expand Up @@ -3023,7 +3016,7 @@ public void FinishWriting()
}
using (SqlBulkCopy sqlBulkCopier = new SqlBulkCopy(connection))
{
sqlBulkCopier.DestinationTableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName);
sqlBulkCopier.DestinationTableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName);
sqlBulkCopier.BulkCopyTimeout = 0;
int skippedFailedRowsCount = 0;
try
Expand Down Expand Up @@ -3052,11 +3045,22 @@ public void FinishWriting()
}
}

public int DeleteExcessFromMainTable(string shop, SqlTransaction transaction, Dictionary<string, Mapping> mappings)
{
sqlCommand.Transaction = transaction;
var mapping = mappings.Values.FirstOrDefault();
if (mapping is null)
return 0;

string extraConditions = GetExtraConditions(mapping, shop, null);
return DeleteRowsFromMainTable(false, mappings, extraConditions, sqlCommand);
}

public void DeleteExcessFromMainTable(string shop, SqlTransaction transaction, string languageId, bool deleteProductsAndGroupForSpecificLanguage, bool hideDeactivatedProducts)
{
foreach (Mapping mapping in job.Mappings.Where(m => !_addedMappingsForMoveToMainTables.Contains(m)))
{
string tempTablePrefix = "TempTableForBulkImport" + mapping.GetId();
string tempTablePrefix = TempTableName + mapping.GetId();
if (HasRowsToImport(mapping, out tempTablePrefix))
{
if ((mapping.DestinationTable.Name == "EcomProducts" || mapping.DestinationTable.Name == "EcomGroups") && deleteProductsAndGroupForSpecificLanguage)
Expand Down Expand Up @@ -3106,7 +3110,7 @@ public void DeleteExistingFromMainTable(string shop, SqlTransaction transaction,
sqlCommand.Transaction = transaction;
foreach (Mapping mapping in job.Mappings)
{
string tempTablePrefix = "TempTableForBulkImport" + mapping.GetId();
string tempTablePrefix = TempTableName + mapping.GetId();
if (HasRowsToImport(mapping, out tempTablePrefix))
{
var rowsAffected = DeleteExistingFromMainTable(sqlCommand, mapping, GetExtraConditions(mapping, shop, languageId), tempTablePrefix);
Expand Down Expand Up @@ -3174,7 +3178,7 @@ public void MoveDataToMainTables(string shop, SqlTransaction sqlTransaction, boo
var mappingId = mapping.GetId();
if (mapping.Active && ColumnMappingsByMappingId[mappingId].Count > 0)
{
string tempTablePrefix = "TempTableForBulkImport" + mappingId;
string tempTablePrefix = TempTableName + mappingId;
if (HasRowsToImport(mapping, out tempTablePrefix))
{
bool? optionValue = mapping.GetOptionValue("UpdateOnlyExistingRecords");
Expand Down Expand Up @@ -4153,7 +4157,7 @@ private Dictionary<int, IEnumerable<string>> GetEcomProductsPKColumns()
{
foreach (DataTable table in DataToWrite.Tables)
{
string tableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName);
string tableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName);
sqlCommand.CommandText = $"if exists (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'{tableName}') AND type in (N'U')) drop table [{tableName}]";
sqlCommand.ExecuteNonQuery();
}
Expand Down Expand Up @@ -4213,7 +4217,7 @@ private void RemoveExcessFromRelationsTables(SqlTransaction sqlTransaction)

foreach (DataTable table in FindDataTablesStartingWithName("EcomVariantOptionsProductRelation"))
{
string tempTableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName);
string tempTableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName);
if (deleteExcess || removeMissingAfterImportDestinationTablesOnly)
{
sqlCommand.CommandText += $"delete from EcomVariantOptionsProductRelation where VariantOptionsProductRelationProductID in (select VariantOptionsProductRelationProductID from {tempTableName}); ";
Expand All @@ -4228,15 +4232,15 @@ private void RemoveExcessFromRelationsTables(SqlTransaction sqlTransaction)

foreach (DataTable table in FindDataTablesStartingWithName("EcomShops"))
{
string tempTableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName);
string tempTableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName);
sqlCommand.CommandText += $"insert into EcomShops (ShopID,ShopName) select shopid,shopname from {tempTableName}; ";
}

sqlCommand.CommandText += "insert into EcomProductsRelatedGroups (RelatedGroupID,RelatedGroupName,RelatedGroupLanguageID) select RelatedGroupID,RelatedGroupName,RelatedGroupLanguageID from EcomProductsRelatedGroupsTempTableForBulkImport; ";

foreach (DataTable table in FindDataTablesStartingWithName("EcomProductsRelated"))
{
string tempTableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName);
string tempTableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName);
sqlCommand.CommandText += "delete from related from EcomProductsRelated related where ProductRelatedProductID in " +
$"(select ProductRelatedProductID from {tempTableName} inside WHERE related.ProductRelatedProductID = inside.ProductRelatedProductID AND " +
"related.ProductRelatedProductRelID = inside.ProductRelatedProductRelID AND related.ProductRelatedGroupID = inside.ProductRelatedGroupID AND related.ProductRelatedProductRelVariantID = inside.ProductRelatedProductRelVariantID); ";
Expand Down Expand Up @@ -4272,7 +4276,7 @@ private void DeleteExcessFromGroupProductRelation(string shop, SqlTransaction sq
sqlClean = new StringBuilder();
foreach (DataTable table in FindDataTablesStartingWithName("EcomProducts"))
{
string tempTableName = GetTableNameWithoutPrefix(table.TableName) + "TempTableForBulkImport" + GetPrefixFromTableName(table.TableName);
string tempTableName = GetTableNameWithoutPrefix(table.TableName) + TempTableName + GetPrefixFromTableName(table.TableName);
sqlClean.Append($"delete EcomGroupProductRelation from {tempTableName} join ecomgroupproductrelation on {tempTableName}.productid=ecomgroupproductrelation.GroupProductRelationProductID where not exists (select * from [dbo].[EcomGroupProductRelationTempTableForBulkImport] where [dbo].[EcomGroupProductRelation].[GroupProductRelationProductID]=[GroupProductRelationProductID] and [dbo].[EcomGroupProductRelation].[GroupProductRelationGroupID]=[GroupProductRelationGroupID] );");
}
}
Expand Down Expand Up @@ -4310,7 +4314,7 @@ private void DeleteExcessFromGroupProductRelation(string shop, SqlTransaction sq
private bool HasRowsToImport(Mapping? mapping, out string tempTablePrefix)
{
bool result = false;
tempTablePrefix = "TempTableForBulkImport" + mapping?.GetId();
tempTablePrefix = TempTableName + mapping?.GetId();

if (mapping != null && mapping.DestinationTable != null && mapping.DestinationTable.Name != null && DataToWrite != null && DataToWrite.Tables != null)
{
Expand All @@ -4322,7 +4326,7 @@ private bool HasRowsToImport(Mapping? mapping, out string tempTablePrefix)
}
else if (DataRowsToWrite.TryGetValue(mapping.DestinationTable.Name, out rows) && rows.Values.Count > 0)
{
tempTablePrefix = "TempTableForBulkImport";
tempTablePrefix = TempTableName;
result = true;
}
}
Expand Down
Loading