Skip to content

Remove missing rows - respect entire table, work so far, need a relea… #52

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
frederik5480 merged 6 commits into from
Jan 9, 2025
Merged

Remove missing rows - respect entire table, work so far, need a relea… #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8860eb8
Remove missing rows - respect entire table, work so far, need a relea…
MatthiasSort Sep 30, 2024
fbc192f
added tooltil for both new addin and updated the old-addin
MatthiasSort Dec 19, 2024
26c325f
updated Serialize()
MatthiasSort Dec 19, 2024
a90f686
merged master into branch to handle conflicts
MatthiasSort Dec 19, 2024
04d8eb1
fixing inlines in .csproj
MatthiasSort Dec 19, 2024
43a1f51
inlines again
MatthiasSort Dec 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 28 additions & 28 deletions src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,30 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Version>10.8.1</Version>
<AssemblyVersion>1.0.0.0</AssemblyVersion>
<Title>Dynamicweb Provider</Title>
<Description>Dynamicweb Provider</Description>
</PropertyGroup>
<PropertyGroup>
<PackageProjectUrl>https://doc.dynamicweb.com/</PackageProjectUrl>
<PackageTags>Dynamicweb dw10 addin integration providers</PackageTags>
<Product>Dynamicweb CMS</Product>
<Company>Dynamicweb Software A/S</Company>
<Authors>Dynamicweb Software A/S</Authors>
<Copyright>Copyright © 2023 Dynamicweb Software A/S</Copyright>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<IncludeSymbols>true</IncludeSymbols>
<EmbedUntrackedSources>true</EmbedUntrackedSources>
<ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>
<PublishRepositoryUrl>true</PublishRepositoryUrl>
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dynamicweb.DataIntegration" Version="10.8.0" />
<PackageReference Include="Dynamicweb.Ecommerce" Version="10.8.0" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
</ItemGroup>
<PropertyGroup>
<Version>10.9.0</Version>
<AssemblyVersion>1.0.0.0</AssemblyVersion>
<Title>Dynamicweb Provider</Title>
<Description>Dynamicweb Provider</Description>
</PropertyGroup>
<PropertyGroup>
<PackageProjectUrl>https://doc.dynamicweb.com/</PackageProjectUrl>
<PackageTags>Dynamicweb dw10 addin integration providers</PackageTags>
<Product>Dynamicweb CMS</Product>
<Company>Dynamicweb Software A/S</Company>
<Authors>Dynamicweb Software A/S</Authors>
<Copyright>Copyright © 2023 Dynamicweb Software A/S</Copyright>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<IncludeSymbols>true</IncludeSymbols>
<EmbedUntrackedSources>true</EmbedUntrackedSources>
<ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>
<PublishRepositoryUrl>true</PublishRepositoryUrl>
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dynamicweb.DataIntegration" Version="10.9.0" />
<PackageReference Include="Dynamicweb.Ecommerce" Version="10.9.0" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
</ItemGroup>
</Project>
12 changes: 12 additions & 0 deletions src/DynamicwebBulkInsertDestinationWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ public class DynamicwebBulkInsertDestinationWriter : BaseSqlWriter, IDestination
protected readonly bool SkipFailingRows;
public SqlCommand SqlCommand;

internal string GetTempTableName
{
get => tempTablePrefix;
}

public new Mapping Mapping { get; }

private DataTable existingUsers;
Expand Down Expand Up @@ -324,6 +329,13 @@ internal int DeleteExcessFromMainTable(string shop, SqlTransaction transaction,
return result;
}

internal int DeleteExcessFromMainTable(string shop, SqlTransaction transaction, Dictionary<string, Mapping> mappings)
{
SqlCommand.Transaction = transaction;
string extraConditions = GetExtraConditions(Mapping, shop, null);
return DeleteRowsFromMainTable(false, mappings, extraConditions, SqlCommand);
}

internal int DeleteExistingFromMainTable(string shop, SqlTransaction transaction)
{
SqlCommand.Transaction = transaction;
Expand Down
51 changes: 45 additions & 6 deletions src/DynamicwebProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,14 @@ public string DefaultLanguage
[AddInParameter("Remove missing rows after import in the destination tables only"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows not present in the import source - excluding related tabled"), AddInParameterGroup("Destination"), AddInParameterOrder(35)]
public bool RemoveMissingAfterImportDestinationTablesOnly { get; set; }

[AddInParameter("Remove missing rows after import"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows not present in the import source - including related tables. This option takes precedence. When Delete incoming rows is ON, this option is ignored"), AddInParameterGroup("Destination"), AddInParameterOrder(40)]
[AddInParameter("Remove missing rows after import"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows from each destination table individually, based on whether they are present in the corresponding source table. This setting looks at each table separately and removes rows missing from the source for that specific table. When Delete incoming rows is ON, this option is ignored"), AddInParameterGroup("Destination"), AddInParameterOrder(40)]
public bool RemoveMissingAfterImport { get; set; }

[AddInParameter("Remove missing rows across all tables after import")]
[AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes rows from all destination tables and relation tables by considering the entire dataset in the import source. This setting evaluates all tables collectively and removes rows missing across the whole activity.")]
[AddInParameterGroup("Destination"), AddInParameterOrder(45)]
public bool RemoveMissingRows { get; set; }

[AddInParameter("Delete incoming rows"), AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=Deletes existing rows present in the import source. When Delete incoming rows is ON, the following options are skipped: Update only existing records, Deactivate missing products, Remove missing rows after import, Delete products / groups for languages included in input, Hide deactivated products"), AddInParameterGroup("Destination"), AddInParameterOrder(50)]
public bool DeleteIncomingItems { get; set; }

Expand Down Expand Up @@ -210,6 +215,7 @@ public string DefaultLanguage

public override void SaveAsXml(XmlTextWriter xmlTextWriter)
{
xmlTextWriter.WriteElementString("RemoveMissingRows", RemoveMissingRows.ToString(CultureInfo.CurrentCulture));
xmlTextWriter.WriteElementString("RemoveMissingAfterImport", RemoveMissingAfterImport.ToString(CultureInfo.CurrentCulture));
xmlTextWriter.WriteElementString("RemoveMissingAfterImportDestinationTablesOnly", RemoveMissingAfterImportDestinationTablesOnly.ToString(CultureInfo.CurrentCulture));
xmlTextWriter.WriteElementString("DeactivateMissingProducts", DeactivateMissingProducts.ToString(CultureInfo.CurrentCulture));
Expand Down Expand Up @@ -257,6 +263,12 @@ public DynamicwebProvider(XmlNode xmlNode)
case "Schema":
Schema = new Schema(node);
break;
case "RemoveMissingRows":
if (node.HasChildNodes)
{
RemoveMissingRows = node.FirstChild.Value == "True";
}
break;
case "RemoveMissingAfterImport":
if (node.HasChildNodes)
{
Expand Down Expand Up @@ -383,6 +395,7 @@ public override void UpdateDestinationSettings(IDestination destination)
DiscardDuplicates = newProvider.DiscardDuplicates;
HideDeactivatedProducts = newProvider.HideDeactivatedProducts;
SkipFailingRows = newProvider.SkipFailingRows;
RemoveMissingRows = newProvider.RemoveMissingRows;
}

public override Schema GetOriginalSourceSchema()
Expand Down Expand Up @@ -585,14 +598,39 @@ public override bool RunJob(Job job)
}
}

foreach (DynamicwebBulkInsertDestinationWriter writer in Enumerable.Reverse(Writers))
if (RemoveMissingRows)
{
bool? optionValue = writer.Mapping.GetOptionValue("DeleteIncomingItems");
bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems;
var distinctWriters = Enumerable.Reverse(Writers).DistinctBy(obj => obj.Mapping.DestinationTable);
if (distinctWriters != null)
{
foreach (var distinctWriter in distinctWriters)
{
if (distinctWriter == null || distinctWriter.Mapping == null)
continue;

var sameWriters = Writers.Where(obj => obj.Mapping != null && obj.Mapping.DestinationTable != null && obj.Mapping.DestinationTable.Name.Equals(distinctWriter.Mapping.DestinationTable?.Name ?? "", StringComparison.OrdinalIgnoreCase)).ToList();
if (sameWriters.Count == 0)
continue;

if (!deleteIncomingItems && writer.RowsToWriteCount > 0)
Dictionary<string, Mapping> mappings = sameWriters.ToDictionary(obj => $"{obj.GetTempTableName}", obj => obj.Mapping);
if (mappings == null || mappings.Count == 0)
continue;

TotalRowsAffected += sameWriters[0].DeleteExcessFromMainTable(Shop, sqlTransaction, mappings);
}
}
}
else
{
foreach (DynamicwebBulkInsertDestinationWriter writer in Enumerable.Reverse(Writers))
{
TotalRowsAffected += writer.DeleteExcessFromMainTable(Shop, sqlTransaction, DeleteProductsAndGroupForSpecificLanguage, defaultLanguage, HideDeactivatedProducts);
bool? optionValue = writer.Mapping.GetOptionValue("DeleteIncomingItems");
bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems;

if (!deleteIncomingItems && writer.RowsToWriteCount > 0)
{
TotalRowsAffected += writer.DeleteExcessFromMainTable(Shop, sqlTransaction, DeleteProductsAndGroupForSpecificLanguage, defaultLanguage, HideDeactivatedProducts);
}
}
}

Expand Down Expand Up @@ -710,6 +748,7 @@ public override string Serialize()
root.Add(CreateParameterNode(GetType(), "Deactivate missing products", DeactivateMissingProducts.ToString()));
root.Add(CreateParameterNode(GetType(), "Update only existing records", UpdateOnlyExistingRecords.ToString()));
root.Add(CreateParameterNode(GetType(), "Insert only new records", InsertOnlyNewRecords.ToString()));
root.Add(CreateParameterNode(GetType(), "Remove missing rows across all tables after import", RemoveMissingRows.ToString()));
root.Add(CreateParameterNode(GetType(), "Remove missing rows after import", RemoveMissingAfterImport.ToString()));
root.Add(CreateParameterNode(GetType(), "Remove missing rows after import in the destination tables only", RemoveMissingAfterImportDestinationTablesOnly.ToString()));
root.Add(CreateParameterNode(GetType(), "Delete incoming rows", DeleteIncomingItems.ToString()));
Expand Down
Loading