Skip to content

Conditional 'OR' is not honored #56

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DWDBE merged 2 commits into from
Feb 26, 2025
Merged

Conditional 'OR' is not honored #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
933ab8e
Conditional 'OR' is not honored
DWDBE Feb 26, 2025
142d640
fix code
DWDBE Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Version>10.8.6</Version>
<Version>10.8.7</Version>
<AssemblyVersion>1.0.0.0</AssemblyVersion>
<Title>OData Provider</Title>
<Description>The Odata Provider lets you fetch and map data from or to any OData endpoint.</Description>
Expand Down
300 changes: 182 additions & 118 deletions src/ODataSourceReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,8 @@
var previusMapping = _mapping.Job.Mappings.Where(mapping => (mapping.SourceTable?.Name ?? "").Equals(_mapping.SourceTable?.Name ?? "", StringComparison.OrdinalIgnoreCase) && mapping.GetId() < _mapping.GetId())?.OrderByDescending(obj => obj.GetId())?.FirstOrDefault();
if (previusMapping != null)
{
var previusMappingFilters = GetFilterAsParameters(previusMapping);
var currentMappingFilters = GetFilterAsParameters(_mapping);
var previusMappingFilters = GetFilterParameters(previusMapping);
var currentMappingFilters = GetFilterParameters(_mapping);
if (previusMappingFilters.Except(currentMappingFilters, StringComparer.OrdinalIgnoreCase).Concat(currentMappingFilters.Except(previusMappingFilters, StringComparer.OrdinalIgnoreCase)).Any())
{
CallEndpoing(headers, readFromLastRequestResponse);
Expand Down Expand Up @@ -165,20 +165,20 @@

var selectAsParameters = GetSelectAsParameters(readFromLastRequestResponse);
var modeAsParemters = GetModeAsParameters();
var filterAsParameters = GetFilterAsParameters(_mapping);
var filterAsParameters = GetFilterStatement(_mapping);
if (!string.IsNullOrEmpty(modeAsParemters))
{
filterAsParameters.Add(modeAsParemters);
filterAsParameters += string.IsNullOrEmpty(filterAsParameters) ? modeAsParemters : $" and {modeAsParemters}";
}

if (selectAsParameters.Any())
{
parameters.Add("$select", string.Join(",", selectAsParameters));
}

if (filterAsParameters.Any())
if (!string.IsNullOrEmpty(filterAsParameters))
{
parameters.Add("$filter", string.Join(" and ", filterAsParameters));
parameters.Add("$filter", filterAsParameters);
}

if (_endpoint.Parameters != null)
Expand Down Expand Up @@ -370,134 +370,198 @@
return result;
}

private List<string> GetFilterAsParameters(Mapping mapping)
private List<string> GetFilterParameters(Mapping mapping)
{
List<string> result = new();
var mappingConditionals = mapping.Conditionals.ToList();

var groups = mapping.Conditionals.Groups.ToList();
if (groups.Count > 0)
{
foreach (var group in groups)
{
var filters = GetFilterAsParameters(mapping, group.Conditionals);
result.AddRange(filters);
}
}
else
{
result = GetFilterAsParameters(mapping, mapping.Conditionals);
}
return result;
}

private string GetFilterStatement(Mapping mapping)
{
string result = "";
var mappingConditionals = mapping.Conditionals.ToList();

var groups = mapping.Conditionals.Groups.ToList();
if (groups.Count > 0)
{
string prependOperator = " And ";
foreach (var group in groups)
{
var filterAsParameters = GetFilterAsParameters(mapping, group.Conditionals);
if (filterAsParameters.Any())
{
var groupFilter = string.Join($" {group.ConditionalOperator} ", filterAsParameters);
result += $"({groupFilter}){prependOperator}";
}
}
if (!string.IsNullOrEmpty(result) && result.Length > prependOperator.Length)
{
result = result.Substring(0, result.Length - prependOperator.Length);
}
}
else
{
var filterAsParameters = GetFilterAsParameters(mapping, mapping.Conditionals);
if (filterAsParameters.Any())
{
result = string.Join(" and ", filterAsParameters);
}
}
return result;
}

private List<string> GetFilterAsParameters(Mapping mapping, IEnumerable<MappingConditional> mappingConditionals)
{
List<string> result = new();
if (mappingConditionals.Any())
{
foreach (var item in mappingConditionals)
{
if (item.Condition != null)
var itemFilter = GetFilterAsParameter(mapping, item);
if (!string.IsNullOrEmpty(itemFilter))
{
string condition = item.Condition;
string operatorInOData = "";
switch (item.ConditionalOperator)
{
case ConditionalOperator.EqualTo:
operatorInOData = "eq";
break;
case ConditionalOperator.LessThan:
operatorInOData = "lt";
break;
case ConditionalOperator.GreaterThan:
operatorInOData = "gt";
break;
case ConditionalOperator.DifferentFrom:
operatorInOData = "ne";
break;
case ConditionalOperator.Contains:
if (item.SourceColumn.Type == typeof(string))
{
result.Add($"contains({item.SourceColumn.Name},'{item.Condition}')");
}
else
{
LogWarningForConditional(item);
}
continue;
case ConditionalOperator.NotContains:
if (item.SourceColumn.Type == typeof(string))
{
result.Add($"contains({item.SourceColumn.Name},'{item.Condition}') ne true");
}
else
{
LogWarningForConditional(item);
}
continue;
case ConditionalOperator.In:
operatorInOData = "eq";
List<string> equalConditions = item.Condition.Split(',').Select(val => val.Trim()).ToList();
if (item.SourceColumn.Type == typeof(string))
{
condition = $"{string.Join($"' or {item.SourceColumn.Name} eq '", equalConditions)}";
}
else
{
condition = $"{string.Join($" or {item.SourceColumn.Name} eq ", equalConditions)}";
}
break;
case ConditionalOperator.NotIn:
operatorInOData = "ne";
List<string> notEqualConditions = item.Condition.Split(',').Select(val => val.Trim()).ToList();
if (item.SourceColumn.Type == typeof(string))
{
condition = $"{string.Join($"' and {item.SourceColumn.Name} ne '", notEqualConditions)}";
}
else
{
condition = $"{string.Join($" and {item.SourceColumn.Name} ne ", notEqualConditions)}";
}
break;
case ConditionalOperator.StartsWith:
if (item.SourceColumn.Type == typeof(string))
{
result.Add($"startswith({item.SourceColumn.Name},'{item.Condition}')");
}
else
{
LogWarningForConditional(item);
}
continue;
case ConditionalOperator.NotStartsWith:
if (item.SourceColumn.Type == typeof(string))
{
result.Add($"startswith({item.SourceColumn.Name},'{item.Condition}') ne true");
}
else
{
LogWarningForConditional(item);
}
continue;
case ConditionalOperator.EndsWith:
if (item.SourceColumn.Type == typeof(string))
{
result.Add($"endswith({item.SourceColumn.Name},'{item.Condition}')");
}
else
{
LogWarningForConditional(item);
}
continue;
case ConditionalOperator.NotEndsWith:
if (item.SourceColumn.Type == typeof(string))
{
result.Add($"endswith({item.SourceColumn.Name},'{item.Condition}') ne true");
}
else
{
LogWarningForConditional(item);
}
continue;
result.Add(itemFilter);
}
}
}
return result;
}

private string GetFilterAsParameter(Mapping mapping, MappingConditional item)
{
string result = "";
if (item.Condition != null)
{
string condition = item.Condition;
string operatorInOData = "";
switch (item.ConditionalOperator)
{
case ConditionalOperator.EqualTo:
operatorInOData = "eq";
break;
case ConditionalOperator.LessThan:
operatorInOData = "lt";
break;
case ConditionalOperator.GreaterThan:
operatorInOData = "gt";
break;
case ConditionalOperator.DifferentFrom:
operatorInOData = "ne";
break;
case ConditionalOperator.Contains:
if (item.SourceColumn.Type == typeof(string))
{
result = $"contains({item.SourceColumn.Name},'{item.Condition}')";
}
else
{
LogWarningForConditional(item);
}
var conditionToAdd = $"{condition}";
return result;
case ConditionalOperator.NotContains:
if (item.SourceColumn.Type == typeof(string))
{
conditionToAdd = $"'{condition}'";
result = $"contains({item.SourceColumn.Name},'{item.Condition}') ne true";
}
else if (item.SourceColumn.Type == typeof(bool))
else
{
conditionToAdd = $"{condition.ToLower()}";
LogWarningForConditional(item);
}
result.Add($"({item.SourceColumn.Name} {operatorInOData} {conditionToAdd})");
}
else
{
_logger?.Warn($"The condition for the table mapping {_mapping.SourceTable.Name} to {_mapping.DestinationTable.Name} on source column {item.SourceColumn.Name} is null, so this have been removed from the $filter.");
}
return result;
case ConditionalOperator.In:
operatorInOData = "eq";
List<string> equalConditions = item.Condition.Split(',').Select(val => val.Trim()).ToList();
if (item.SourceColumn.Type == typeof(string))
{
condition = $"{string.Join($"' or {item.SourceColumn.Name} eq '", equalConditions)}";
}
else
{
condition = $"{string.Join($" or {item.SourceColumn.Name} eq ", equalConditions)}";
}
break;
case ConditionalOperator.NotIn:
operatorInOData = "ne";
List<string> notEqualConditions = item.Condition.Split(',').Select(val => val.Trim()).ToList();
if (item.SourceColumn.Type == typeof(string))
{
condition = $"{string.Join($"' and {item.SourceColumn.Name} ne '", notEqualConditions)}";
}
else
{
condition = $"{string.Join($" and {item.SourceColumn.Name} ne ", notEqualConditions)}";
}
break;
case ConditionalOperator.StartsWith:
if (item.SourceColumn.Type == typeof(string))
{
result = $"startswith({item.SourceColumn.Name},'{item.Condition}')";
}
else
{
LogWarningForConditional(item);
}
return result;
case ConditionalOperator.NotStartsWith:
if (item.SourceColumn.Type == typeof(string))
{
result = $"startswith({item.SourceColumn.Name},'{item.Condition}') ne true";
}
else
{
LogWarningForConditional(item);
}
return result;
case ConditionalOperator.EndsWith:
if (item.SourceColumn.Type == typeof(string))
{
result = $"endswith({item.SourceColumn.Name},'{item.Condition}')";
}
else
{
LogWarningForConditional(item);
}
return result;
case ConditionalOperator.NotEndsWith:
if (item.SourceColumn.Type == typeof(string))
{
result = $"endswith({item.SourceColumn.Name},'{item.Condition}') ne true";
}
else
{
LogWarningForConditional(item);
}
return result;
}
var conditionToAdd = $"{condition}";
if (item.SourceColumn.Type == typeof(string))
{
conditionToAdd = $"'{condition}'";
}
else if (item.SourceColumn.Type == typeof(bool))
{
conditionToAdd = $"{condition.ToLower()}";
}
result = $"({item.SourceColumn.Name} {operatorInOData} {conditionToAdd})";
}
else
{
_logger?.Warn($"The condition for the table mapping {_mapping.SourceTable.Name} to {_mapping.DestinationTable.Name} on source column {item.SourceColumn.Name} is null, so this have been removed from the $filter.");
}
return result;
}
Expand Down Expand Up @@ -580,11 +644,11 @@
if (endpointAuthentication.IsTokenBased())
{
string token = GetToken(_endpoint, endpointAuthentication);
task = RetryHelper.RetryOnExceptionAsync<Exception>(10, async () => { _httpRestClient.GetAsync(url, HandleStream, token, (Dictionary<string, string>)headers).Wait(new CancellationTokenSource(_timeoutInMilliseconds).Token); }, _logger);

Check warning on line 647 in src/ODataSourceReader.cs

View workflow job for this annotation

GitHub Actions / call-workflow / Build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 647 in src/ODataSourceReader.cs

View workflow job for this annotation

GitHub Actions / call-workflow / Build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.
}
else
{
task = RetryHelper.RetryOnExceptionAsync<Exception>(10, async () => { _httpRestClient.GetAsync(url, HandleStream, endpointAuthentication, (Dictionary<string, string>)headers).Wait(new CancellationTokenSource(_timeoutInMilliseconds).Token); }, _logger);

Check warning on line 651 in src/ODataSourceReader.cs

View workflow job for this annotation

GitHub Actions / call-workflow / Build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 651 in src/ODataSourceReader.cs

View workflow job for this annotation

GitHub Actions / call-workflow / Build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.
}
if (task.IsCanceled)
{
Expand Down Expand Up @@ -701,11 +765,11 @@
if (endpointAuthentication.IsTokenBased())
{
string token = GetToken(_endpoint, endpointAuthentication);
task = RetryHelper.RetryOnExceptionAsync<Exception>(10, async () => { _httpRestClient.GetAsync(checkUrl, HandleResponse, token, (Dictionary<string, string>)GetAllHeaders()).Wait(new CancellationTokenSource(_timeoutInMilliseconds).Token); }, _logger);

Check warning on line 768 in src/ODataSourceReader.cs

View workflow job for this annotation

GitHub Actions / call-workflow / Build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 768 in src/ODataSourceReader.cs

View workflow job for this annotation

GitHub Actions / call-workflow / Build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.
}
else
{
task = RetryHelper.RetryOnExceptionAsync<Exception>(10, async () => { _httpRestClient.GetAsync(checkUrl, HandleResponse, endpointAuthentication, (Dictionary<string, string>)GetAllHeaders()).Wait(new CancellationTokenSource(_timeoutInMilliseconds).Token); }, _logger);

Check warning on line 772 in src/ODataSourceReader.cs

View workflow job for this annotation

GitHub Actions / call-workflow / Build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 772 in src/ODataSourceReader.cs

View workflow job for this annotation

GitHub Actions / call-workflow / Build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.
}
if (task.IsCanceled)
{
Expand Down