From 433bec42a10e17a9721a34b3afae541e3be37a4f Mon Sep 17 00:00:00 2001 From: Dmitriy Benyuk Date: Tue, 25 Mar 2025 11:23:20 +0200 Subject: [PATCH 1/2] Fix getting sql Connection --- ...ration.Providers.DynamicwebProvider.csproj | 2 +- src/DynamicwebProvider.cs | 20 ++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj index 5fdf9a3..68dd3a6 100644 --- a/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.DynamicwebProvider.csproj @@ -1,6 +1,6 @@  - 10.9.1 + 10.9.2 1.0.0.0 Dynamicweb Provider Dynamicweb Provider diff --git a/src/DynamicwebProvider.cs b/src/DynamicwebProvider.cs index 0b3914e..7101a66 100644 --- a/src/DynamicwebProvider.cs +++ b/src/DynamicwebProvider.cs @@ -63,7 +63,7 @@ public DynamicwebProvider() public DynamicwebProvider(string connectionString) { SqlConnectionString = connectionString; - connection = new SqlConnection(SqlConnectionString); + _connection = new SqlConnection(SqlConnectionString); } #region HideParameters @@ -137,11 +137,11 @@ public string DestinationDatabase } #endregion - protected SqlConnection connection; + protected SqlConnection _connection; protected SqlConnection Connection { - get { return connection ??= (SqlConnection)Database.CreateConnection(); } - set { connection = value; } + get { return _connection ??= (SqlConnection)Database.CreateConnection(); } + set { _connection = value; } } protected string defaultLanguage = null; @@ -460,6 +460,9 @@ public override void OverwriteDestinationSchemaToOriginal() public override bool RunJob(Job job) { + if (Connection.State != ConnectionState.Open) + Connection.Open(); + if (IsFirstJobRun) { OrderTablesByConstraints(job, Connection); @@ -470,9 +473,7 @@ public override bool RunJob(Job job) bool deactivateMissingProducts = false; try { - ReplaceMappingConditionalsWithValuesFromRequest(job); - if (Connection.State != ConnectionState.Open) - Connection.Open(); + ReplaceMappingConditionalsWithValuesFromRequest(job); foreach (Mapping mapping in job.Mappings) { @@ -697,7 +698,8 @@ public override bool RunJob(Job job) writer.Close(); } job.Source.Close(); - Connection.Dispose(); + _connection.Dispose(); + _connection = null; sourceRow = null; } if (IsFirstJobRun) @@ -883,7 +885,7 @@ public IEnumerable GetParameterOptions(string parameterName) public override void Close() { - Connection.Close(); + _connection?.Close(); } public override ISourceReader GetReader(Mapping mapping) From 3266b0c224d938399d4f265173f2c8ba81ba5142 Mon Sep 17 00:00:00 2001 From: Dmitriy Benyuk Date: Tue, 25 Mar 2025 19:57:40 +0200 Subject: [PATCH 2/2] Fix for import --- src/DynamicwebProvider.cs | 196 +++++++++++++++++++++----------------- 1 file changed, 111 insertions(+), 85 deletions(-) diff --git a/src/DynamicwebProvider.cs b/src/DynamicwebProvider.cs index 7101a66..7131ddf 100644 --- a/src/DynamicwebProvider.cs +++ b/src/DynamicwebProvider.cs @@ -469,11 +469,12 @@ public override bool RunJob(Job job) } SqlTransaction sqlTransaction = null; Dictionary sourceRow = null; + bool isReadFromSourceFinished = IsReadingSourceXmlFinished(job); Exception exception = null; bool deactivateMissingProducts = false; try { - ReplaceMappingConditionalsWithValuesFromRequest(job); + ReplaceMappingConditionalsWithValuesFromRequest(job); foreach (Mapping mapping in job.Mappings) { @@ -500,9 +501,9 @@ public override bool RunJob(Job job) { Column randomColumn = mapping.SourceTable.Columns.First(); var languageColumnMapping = mapping.AddMapping(randomColumn, mapping.DestinationTable.Columns.Find(c => string.Compare(c.Name, MappingExtensions.GetLanguageIdColumnName(mapping.DestinationTable.Name), true) == 0)); - languageColumnMapping.ScriptTypeProvider = new ConstantScriptType() - { - ScriptValue = defaultLanguage + languageColumnMapping.ScriptTypeProvider = new ConstantScriptType() + { + ScriptValue = defaultLanguage }; } } @@ -549,115 +550,118 @@ public override bool RunJob(Job job) } sourceRow = null; - sqlTransaction = Connection.BeginTransaction(); - - deactivateMissingProducts = DeactivateMissingProducts; - var productsWriter = Writers.Where(w => w.Mapping != null && w.Mapping.Active && w.Mapping.DestinationTable != null && - w.Mapping.DestinationTable.Name == "EcomProducts").FirstOrDefault(); - if (productsWriter != null) + if (isReadFromSourceFinished) { - bool? value = productsWriter.Mapping.GetOptionValue("DeactivateMissingProducts"); - deactivateMissingProducts = value.HasValue ? value.Value : DeactivateMissingProducts; - } + sqlTransaction = Connection.BeginTransaction(); - if (deactivateMissingProducts) - HandleProductsWriter(Writers); + deactivateMissingProducts = DeactivateMissingProducts; + var productsWriter = Writers.Where(w => w.Mapping != null && w.Mapping.Active && w.Mapping.DestinationTable != null && + w.Mapping.DestinationTable.Name == "EcomProducts").FirstOrDefault(); + if (productsWriter != null) + { + bool? value = productsWriter.Mapping.GetOptionValue("DeactivateMissingProducts"); + deactivateMissingProducts = value.HasValue ? value.Value : DeactivateMissingProducts; + } - foreach (DynamicwebBulkInsertDestinationWriter writer in Writers) - { - bool? optionValue = writer.Mapping.GetOptionValue("DeleteIncomingItems"); - bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems; + if (deactivateMissingProducts) + HandleProductsWriter(Writers); - if (writer.RowsToWriteCount > 0) + foreach (DynamicwebBulkInsertDestinationWriter writer in Writers) { - if (deleteIncomingItems) + bool? optionValue = writer.Mapping.GetOptionValue("DeleteIncomingItems"); + bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems; + + if (writer.RowsToWriteCount > 0) { - int rowsAffected = writer.DeleteExistingFromMainTable(Shop, sqlTransaction); - if (rowsAffected > 0) + if (deleteIncomingItems) { - Logger.Log($"The number of deleted rows: {rowsAffected} for the destination {writer.Mapping.DestinationTable.Name} table mapping"); - TotalRowsAffected += rowsAffected; + int rowsAffected = writer.DeleteExistingFromMainTable(Shop, sqlTransaction); + if (rowsAffected > 0) + { + Logger.Log($"The number of deleted rows: {rowsAffected} for the destination {writer.Mapping.DestinationTable.Name} table mapping"); + TotalRowsAffected += rowsAffected; + } } - } - else - { - writer.AddMappingsToJobThatNeedsToBeThereForMoveToMainTables(job); - DynamicwebBulkInsertDestinationWriter.RemoveColumnMappingsFromJobThatShouldBeSkippedInMoveToMainTables(job); + else + { + writer.AddMappingsToJobThatNeedsToBeThereForMoveToMainTables(job); + DynamicwebBulkInsertDestinationWriter.RemoveColumnMappingsFromJobThatShouldBeSkippedInMoveToMainTables(job); - optionValue = writer.Mapping.GetOptionValue("UpdateOnlyExistingRecords"); - bool updateOnlyExistingRecords = optionValue.HasValue ? optionValue.Value : UpdateOnlyExistingRecords; + optionValue = writer.Mapping.GetOptionValue("UpdateOnlyExistingRecords"); + bool updateOnlyExistingRecords = optionValue.HasValue ? optionValue.Value : UpdateOnlyExistingRecords; - int rowsAffected = writer.MoveDataToMainTable(sqlTransaction, updateOnlyExistingRecords, InsertOnlyNewRecords); - if (rowsAffected > 0) - { - Logger.Log($"The number of rows affected: {rowsAffected} in the {writer.Mapping.DestinationTable.Name} table"); - TotalRowsAffected += rowsAffected; + int rowsAffected = writer.MoveDataToMainTable(sqlTransaction, updateOnlyExistingRecords, InsertOnlyNewRecords); + if (rowsAffected > 0) + { + Logger.Log($"The number of rows affected: {rowsAffected} in the {writer.Mapping.DestinationTable.Name} table"); + TotalRowsAffected += rowsAffected; + } } } - } - else - { - if (!deleteIncomingItems) + else { - Logger.Log(string.Format("No rows were imported to the table: {0}.", writer.Mapping.DestinationTable.Name)); + if (!deleteIncomingItems) + { + Logger.Log(string.Format("No rows were imported to the table: {0}.", writer.Mapping.DestinationTable.Name)); + } } } - } - if (RemoveMissingRows) - { - var distinctWriters = Enumerable.Reverse(Writers).DistinctBy(obj => obj.Mapping.DestinationTable); - if (distinctWriters != null) + if (RemoveMissingRows) { - foreach (var distinctWriter in distinctWriters) + var distinctWriters = Enumerable.Reverse(Writers).DistinctBy(obj => obj.Mapping.DestinationTable); + if (distinctWriters != null) { - if (distinctWriter == null || distinctWriter.Mapping == null) - continue; + foreach (var distinctWriter in distinctWriters) + { + if (distinctWriter == null || distinctWriter.Mapping == null) + continue; - var sameWriters = Writers.Where(obj => obj.Mapping != null && obj.Mapping.DestinationTable != null && obj.Mapping.DestinationTable.Name.Equals(distinctWriter.Mapping.DestinationTable?.Name ?? "", StringComparison.OrdinalIgnoreCase)).ToList(); - if (sameWriters.Count == 0) - continue; + var sameWriters = Writers.Where(obj => obj.Mapping != null && obj.Mapping.DestinationTable != null && obj.Mapping.DestinationTable.Name.Equals(distinctWriter.Mapping.DestinationTable?.Name ?? "", StringComparison.OrdinalIgnoreCase)).ToList(); + if (sameWriters.Count == 0) + continue; - Dictionary mappings = sameWriters.ToDictionary(obj => $"{obj.GetTempTableName}", obj => obj.Mapping); - if (mappings == null || mappings.Count == 0) - continue; + Dictionary mappings = sameWriters.ToDictionary(obj => $"{obj.GetTempTableName}", obj => obj.Mapping); + if (mappings == null || mappings.Count == 0) + continue; - TotalRowsAffected += sameWriters[0].DeleteExcessFromMainTable(Shop, sqlTransaction, mappings); + TotalRowsAffected += sameWriters[0].DeleteExcessFromMainTable(Shop, sqlTransaction, mappings); + } } } - } - else - { - foreach (DynamicwebBulkInsertDestinationWriter writer in Enumerable.Reverse(Writers)) + else { - bool? optionValue = writer.Mapping.GetOptionValue("DeleteIncomingItems"); - bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems; - - if (!deleteIncomingItems && writer.RowsToWriteCount > 0) + foreach (DynamicwebBulkInsertDestinationWriter writer in Enumerable.Reverse(Writers)) { - TotalRowsAffected += writer.DeleteExcessFromMainTable(Shop, sqlTransaction, DeleteProductsAndGroupForSpecificLanguage, defaultLanguage, HideDeactivatedProducts); + bool? optionValue = writer.Mapping.GetOptionValue("DeleteIncomingItems"); + bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems; + + if (!deleteIncomingItems && writer.RowsToWriteCount > 0) + { + TotalRowsAffected += writer.DeleteExcessFromMainTable(Shop, sqlTransaction, DeleteProductsAndGroupForSpecificLanguage, defaultLanguage, HideDeactivatedProducts); + } } } - } - if (PartialUpdate) - { - //if PartilUpdate property is set, we still want to remove rows from the EcomGroupProductRelationsTable, but only for the products that are being imported - DynamicwebBulkInsertDestinationWriter groupProductRelationWriter = Writers.Find(w => w.Mapping.DestinationTable != null && w.Mapping.DestinationTable.Name == "EcomGroupProductRelation"); - if (groupProductRelationWriter != null && groupProductRelationWriter.RowsToWriteCount > 0) + if (PartialUpdate) { - bool? optionValue = groupProductRelationWriter.Mapping.GetOptionValue("DeleteIncomingItems"); - bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems; - if (!deleteIncomingItems) + //if PartilUpdate property is set, we still want to remove rows from the EcomGroupProductRelationsTable, but only for the products that are being imported + DynamicwebBulkInsertDestinationWriter groupProductRelationWriter = Writers.Find(w => w.Mapping.DestinationTable != null && w.Mapping.DestinationTable.Name == "EcomGroupProductRelation"); + if (groupProductRelationWriter != null && groupProductRelationWriter.RowsToWriteCount > 0) { - TotalRowsAffected += groupProductRelationWriter.DeleteExcessGroupProductsRelationsTable(); + bool? optionValue = groupProductRelationWriter.Mapping.GetOptionValue("DeleteIncomingItems"); + bool deleteIncomingItems = optionValue.HasValue ? optionValue.Value : DeleteIncomingItems; + if (!deleteIncomingItems) + { + TotalRowsAffected += groupProductRelationWriter.DeleteExcessGroupProductsRelationsTable(); + } } } - } - sqlTransaction.Commit(); - AssortmentHandler?.RebuildAssortments(); - MoveRepositoriesIndexToJob(job); + sqlTransaction.Commit(); + AssortmentHandler?.RebuildAssortments(); + MoveRepositoriesIndexToJob(job); + } } catch (Exception ex) { @@ -693,13 +697,16 @@ public override bool RunJob(Job job) } finally { - foreach (var writer in Writers) + if (exception != null || isReadFromSourceFinished) { - writer.Close(); + foreach (var writer in Writers) + { + writer.Close(); + } + job.Source.Close(); + _connection.Dispose(); + _connection = null; } - job.Source.Close(); - _connection.Dispose(); - _connection = null; sourceRow = null; } if (IsFirstJobRun) @@ -909,4 +916,23 @@ IEnumerable IParameterVisibility.GetHiddenParameterNames(string paramete } return result; } + + private bool IsReadingSourceXmlFinished(Job job) + { + if (job.Source != null) + { + bool isXmlProviderUsed = job.Source.GetType().Name.EndsWith("XmlProvider", StringComparison.OrdinalIgnoreCase); + if (isXmlProviderUsed) + { + var exportIsDoneProperty = job.Source.GetType().GetField("ExportIsDone"); + var exportIsDoneValue = exportIsDoneProperty?.GetValue(job.Source); + if (exportIsDoneValue is not null) + { + return Converter.ToBoolean(exportIsDoneValue); + } + } + } + return true; + } + }