diff --git a/src/vtlengine/AST/ASTString.py b/src/vtlengine/AST/ASTString.py index 71ad5ccd0..e70b68e96 100644 --- a/src/vtlengine/AST/ASTString.py +++ b/src/vtlengine/AST/ASTString.py @@ -134,10 +134,10 @@ def visit_HRuleset(self, node: AST.HRuleset) -> None: self.vtl_script += f"define hierarchical ruleset {node.name}({signature}) is{nl}" for _i, rule in enumerate(node.rules): rule_str = f"{tab}{self.visit(rule)}" - if rule.erCode: + if rule.erCode is not None: rule_str += f"{nl}{tab}errorcode {_handle_literal(rule.erCode)}" - if rule.erLevel: - rule_str += f"{nl}{tab}errorlevel {rule.erLevel}" + if rule.erLevel is not None: + rule_str += f"{nl}{tab}errorlevel {_handle_literal(rule.erLevel)}" rules_strs.append(rule_str) rules_sep = f";{nl * 2}" if len(rules_strs) > 1 else "" rules = rules_sep.join(rules_strs) @@ -146,10 +146,10 @@ def visit_HRuleset(self, node: AST.HRuleset) -> None: else: for rule in node.rules: rule_str = self.visit(rule) - if rule.erCode: + if rule.erCode is not None: rule_str += f" errorcode {_handle_literal(rule.erCode)}" - if rule.erLevel: - rule_str += f" errorlevel {rule.erLevel}" + if rule.erLevel is not None: + rule_str += f" errorlevel {_handle_literal(rule.erLevel)}" rules_strs.append(rule_str) rules_sep = "; " if len(rules_strs) > 1 else "" rules = rules_sep.join(rules_strs) @@ -195,7 +195,7 @@ def visit_DPRule(self, node: AST.DPRule) -> str: if node.erCode is not None: lines.append(f"{tab * 3}errorcode {_handle_literal(node.erCode)}") if node.erLevel is not None: - lines.append(f"{tab * 3}errorlevel {node.erLevel}") + lines.append(f"{tab * 3}errorlevel {_handle_literal(node.erLevel)}") return nl.join(lines) else: vtl_script = "" @@ -205,7 +205,7 @@ def visit_DPRule(self, node: AST.DPRule) -> str: if node.erCode is not None: vtl_script += f" errorcode {_handle_literal(node.erCode)}" if node.erLevel is not None: - vtl_script += f" errorlevel {node.erLevel}" + vtl_script += f" errorlevel {_handle_literal(node.erLevel)}" return vtl_script def visit_DPRIdentifier(self, node: AST.DPRIdentifier) -> str: diff --git a/src/vtlengine/AST/__init__.py b/src/vtlengine/AST/__init__.py index 990a54971..36600c2ac 100644 --- a/src/vtlengine/AST/__init__.py +++ b/src/vtlengine/AST/__init__.py @@ -484,8 +484,8 @@ class Validation(AST): op: str validation: AST - error_code: Optional[str] - error_level: Optional[Union[int, str]] + error_code: Optional[Union[str, int, float, bool]] + error_level: Optional[Union[str, int, float, bool]] imbalance: Optional[AST] invalid: bool @@ -631,8 +631,8 @@ class HRule(AST): name: Optional[str] rule: HRBinOp - erCode: Optional[str] - erLevel: Optional[Union[int, str]] + erCode: Optional[Union[str, int, float, bool]] + erLevel: Optional[Union[str, int, float, bool]] __eq__ = AST.ast_equality @@ -645,8 +645,8 @@ class DPRule(AST): name: Optional[str] rule: HRBinOp - erCode: Optional[str] - erLevel: Optional[Union[int, str]] + erCode: Optional[Union[str, int, float, bool]] + erLevel: Optional[Union[str, int, float, bool]] __eq__ = AST.ast_equality diff --git a/src/vtlengine/Operators/Validation.py b/src/vtlengine/Operators/Validation.py index bba9735d4..7c89d6273 100644 --- a/src/vtlengine/Operators/Validation.py +++ b/src/vtlengine/Operators/Validation.py @@ -1,5 +1,5 @@ from copy import copy -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Optional, Type, Union import pandas as pd @@ -8,6 +8,7 @@ Boolean, Integer, Number, + ScalarType, String, check_unary_implicit_promotion, ) @@ -26,8 +27,8 @@ def validate( cls, validation_element: Dataset, imbalance_element: Optional[Dataset], - error_code: Optional[str], - error_level: Optional[Union[int, str]], + error_code: Optional[Union[str, int, float, bool]], + error_level: Optional[Union[str, int, float, bool]], invalid: bool, ) -> Dataset: dataset_name = VirtualCounter._new_ds_name() @@ -36,11 +37,13 @@ def validate( measure = validation_element.get_measures()[0] if measure.data_type != Boolean: raise SemanticError("1-1-10-1", op=cls.op, op_type="validation", me_type="Boolean") - error_level_type = None - if error_level is None or isinstance(error_level, int): + error_level_type: Optional[Type[ScalarType]] = None + if isinstance(error_level, bool): + error_level_type = Boolean + elif error_level is None or isinstance(error_level, int): error_level_type = Integer elif isinstance(error_level, str): - error_level_type = String # type: ignore[assignment] + error_level_type = String else: error_level_type = String @@ -79,7 +82,7 @@ def validate( result_components["errorlevel"] = Component( name="errorlevel", - data_type=error_level_type, # type: ignore[arg-type] + data_type=error_level_type, role=Role.MEASURE, nullable=True, ) @@ -91,8 +94,8 @@ def evaluate( cls, validation_element: Dataset, imbalance_element: Optional[Dataset], - error_code: Optional[str], - error_level: Optional[Union[int, str]], + error_code: Optional[Union[str, int, float, bool]], + error_level: Optional[Union[str, int, float, bool]], invalid: bool, ) -> Dataset: result = cls.validate( @@ -116,7 +119,8 @@ def evaluate( bool_col = result.data[validation_measure_name] is_false = bool_col.fillna(True) == False # noqa: E712 result.data["errorcode"] = pd.Series(None, index=result.data.index, dtype="string[pyarrow]") - result.data.loc[is_false, "errorcode"] = error_code + ec_value = str(error_code) if error_code is not None else None + result.data.loc[is_false, "errorcode"] = ec_value errorlevel_dtype = result.components["errorlevel"].data_type.dtype() result.data["errorlevel"] = pd.Series(None, index=result.data.index, dtype=errorlevel_dtype) if error_level is not None: @@ -150,7 +154,7 @@ def _generate_result_data(cls, rule_info: Dict[str, Any]) -> pd.DataFrame: @classmethod def validate(cls, dataset_element: Dataset, rule_info: Dict[str, Any], output: str) -> Dataset: - error_level_type = None + error_level_type: Optional[Type[ScalarType]] = None error_levels = [ rule_data.get("errorlevel") for rule_data in rule_info.values() @@ -158,12 +162,14 @@ def validate(cls, dataset_element: Dataset, rule_info: Dict[str, Any], output: s ] non_null_levels = [el for el in error_levels if el is not None] - if len(non_null_levels) == 0 or all(isinstance(el, int) for el in non_null_levels): + if all(isinstance(el, bool) for el in non_null_levels) and len(non_null_levels) > 0: + error_level_type = Boolean + elif len(non_null_levels) == 0 or all(isinstance(el, int) for el in non_null_levels): error_level_type = Number elif all(isinstance(el, str) for el in non_null_levels): - error_level_type = String # type: ignore[assignment] + error_level_type = String else: - error_level_type = String # type: ignore[assignment] + error_level_type = String dataset_name = VirtualCounter._new_ds_name() result_components = {comp.name: comp for comp in dataset_element.get_identifiers()} result_components["ruleid"] = Component( @@ -191,7 +197,7 @@ def validate(cls, dataset_element: Dataset, rule_info: Dict[str, Any], output: s ) result_components["errorlevel"] = Component( name="errorlevel", - data_type=error_level_type, # type: ignore[arg-type] + data_type=error_level_type, role=Role.MEASURE, nullable=True, ) diff --git a/tests/DatapointRulesets/data/DataSet/input/GH_598_2-1.csv b/tests/DatapointRulesets/data/DataSet/input/GH_598_2-1.csv new file mode 100644 index 000000000..c93f7a146 --- /dev/null +++ b/tests/DatapointRulesets/data/DataSet/input/GH_598_2-1.csv @@ -0,0 +1,5 @@ +Id_1,Id_2,Me_1 +A,1,10 +A,2,-5 +B,1,100 +B,2,30 diff --git a/tests/DatapointRulesets/data/DataSet/output/GH_598_2-1.csv b/tests/DatapointRulesets/data/DataSet/output/GH_598_2-1.csv new file mode 100644 index 000000000..efa161630 --- /dev/null +++ b/tests/DatapointRulesets/data/DataSet/output/GH_598_2-1.csv @@ -0,0 +1,3 @@ +Id_1,Id_2,ruleid,Me_1,errorcode,errorlevel +A,2,rule_1,-5.0,True,True +B,2,rule_2,30.0,False,False diff --git a/tests/DatapointRulesets/data/DataStructure/input/GH_598_2-1.json b/tests/DatapointRulesets/data/DataStructure/input/GH_598_2-1.json new file mode 100644 index 000000000..a20b58429 --- /dev/null +++ b/tests/DatapointRulesets/data/DataStructure/input/GH_598_2-1.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Integer", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/DatapointRulesets/data/DataStructure/output/GH_598_2-1.json b/tests/DatapointRulesets/data/DataStructure/output/GH_598_2-1.json new file mode 100644 index 000000000..ed586e7c3 --- /dev/null +++ b/tests/DatapointRulesets/data/DataStructure/output/GH_598_2-1.json @@ -0,0 +1,45 @@ +{ + "datasets": [ + { + "name": "DS_r", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Integer", + "nullable": false + }, + { + "name": "ruleid", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "errorcode", + "role": "Measure", + "type": "String", + "nullable": true + }, + { + "name": "errorlevel", + "role": "Measure", + "type": "Boolean", + "nullable": true + } + ] + } + ] +} diff --git a/tests/DatapointRulesets/data/vtl/GH_598_2.vtl b/tests/DatapointRulesets/data/vtl/GH_598_2.vtl new file mode 100644 index 000000000..171db56df --- /dev/null +++ b/tests/DatapointRulesets/data/vtl/GH_598_2.vtl @@ -0,0 +1,6 @@ +define datapoint ruleset dp_bool (variable Id_1 as X, Me_1 as M) is + rule_1: when X = "A" then M > 0 errorcode true errorlevel true; + rule_2: when X = "B" then M > 50 errorcode false errorlevel false +end datapoint ruleset; + +DS_r <- check_datapoint(DS_1, dp_bool); diff --git a/tests/DatapointRulesets/test_datapoint_rulesets.py b/tests/DatapointRulesets/test_datapoint_rulesets.py index 1d8434e0e..bcf51f3ba 100644 --- a/tests/DatapointRulesets/test_datapoint_rulesets.py +++ b/tests/DatapointRulesets/test_datapoint_rulesets.py @@ -1,6 +1,8 @@ from pathlib import Path from tests.Helper import TestHelper +from vtlengine.API import create_ast +from vtlengine.AST.ASTString import ASTString class TestDataPointRuleset(TestHelper): @@ -420,3 +422,44 @@ def test_14(self): self.NewSemanticExceptionTest( code=code, number_inputs=number_inputs, exception_code=message ) + + def test_GH_598_2(self): + """ + check_datapoint with boolean errorcode and errorlevel constants. + Dataset --> Dataset + Status: OK + + define datapoint ruleset dp_bool (variable Id_1 as X, Me_1 as M) is + rule_1: when X = "A" then M > 0 errorcode true errorlevel true; + rule_2: when X = "B" then M > 50 errorcode false errorlevel false + end datapoint ruleset; + + DS_r <- check_datapoint(DS_1, dp_bool); + + Git Branch: cr-596. + Goal: Verify boolean constants work as errorcode/errorlevel in datapoint + rulesets, including AST round-trip. + """ + code = "GH_598_2" + number_inputs = 1 + references_names = ["1"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_598_2_roundtrip(self): + """ + AST round-trip for check_datapoint with boolean errorcode/errorlevel. + + Git Branch: cr-596. + Goal: Verify that rendering the AST back to VTL and re-parsing produces the + same result when boolean constants are used in errorcode/errorlevel. + """ + code = "GH_598_2" + number_inputs = 1 + references_names = ["1"] + + text = self.LoadVTL(code) + rendered = ASTString().render(create_ast(text)) + self.BaseTest( + code=code, number_inputs=number_inputs, references_names=references_names, text=rendered + ) diff --git a/tests/Hierarchical/data/DataSet/input/GH_598_1-1.csv b/tests/Hierarchical/data/DataSet/input/GH_598_1-1.csv new file mode 100644 index 000000000..2e7361820 --- /dev/null +++ b/tests/Hierarchical/data/DataSet/input/GH_598_1-1.csv @@ -0,0 +1,7 @@ +Id_1,Id_2,Me_1 +B,XX,100 +C,XX,80 +D,XX,30 +N,XX,200 +A,XX,190 +L,XX,10 diff --git a/tests/Hierarchical/data/DataSet/output/GH_598_1-1.csv b/tests/Hierarchical/data/DataSet/output/GH_598_1-1.csv new file mode 100644 index 000000000..5a9e87326 --- /dev/null +++ b/tests/Hierarchical/data/DataSet/output/GH_598_1-1.csv @@ -0,0 +1,3 @@ +Id_1,Id_2,ruleid,Me_1,errorcode,errorlevel,imbalance +B,XX,1,100.0,True,True,50.0 +N,XX,2,200.0,False,False,20.0 diff --git a/tests/Hierarchical/data/DataStructure/input/GH_598_1-1.json b/tests/Hierarchical/data/DataStructure/input/GH_598_1-1.json new file mode 100644 index 000000000..50a362378 --- /dev/null +++ b/tests/Hierarchical/data/DataStructure/input/GH_598_1-1.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Hierarchical/data/DataStructure/output/GH_598_1-1.json b/tests/Hierarchical/data/DataStructure/output/GH_598_1-1.json new file mode 100644 index 000000000..5d681c006 --- /dev/null +++ b/tests/Hierarchical/data/DataStructure/output/GH_598_1-1.json @@ -0,0 +1,51 @@ +{ + "datasets": [ + { + "name": "DS_r", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "ruleid", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "errorcode", + "role": "Measure", + "type": "String", + "nullable": true + }, + { + "name": "errorlevel", + "role": "Measure", + "type": "Boolean", + "nullable": true + }, + { + "name": "imbalance", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Hierarchical/data/vtl/GH_598_1.vtl b/tests/Hierarchical/data/vtl/GH_598_1.vtl new file mode 100644 index 000000000..2bb632d28 --- /dev/null +++ b/tests/Hierarchical/data/vtl/GH_598_1.vtl @@ -0,0 +1,6 @@ +define hierarchical ruleset hr_bool (variable rule Id_1) is + B = C - D errorcode true errorlevel true; + N = A - L errorcode false errorlevel false +end hierarchical ruleset; + +DS_r <- check_hierarchy(DS_1, hr_bool rule Id_1); diff --git a/tests/Hierarchical/test_hierarchical.py b/tests/Hierarchical/test_hierarchical.py index 727d4071a..37225fe76 100644 --- a/tests/Hierarchical/test_hierarchical.py +++ b/tests/Hierarchical/test_hierarchical.py @@ -3,6 +3,8 @@ import pytest from tests.Helper import TestHelper +from vtlengine.API import create_ast +from vtlengine.AST.ASTString import ASTString class HierarchicalHelper(TestHelper): @@ -1437,6 +1439,47 @@ def test_GL_566_1(self): self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + def test_GH_598_1(self): + """ + check_hierarchy with boolean errorcode and errorlevel constants. + Dataset --> Dataset + Status: OK + + define hierarchical ruleset hr_bool (variable rule Id_1) is + B = C - D errorcode true errorlevel true; + N = A - L errorcode false errorlevel false + end hierarchical ruleset; + + DS_r <- check_hierarchy(DS_1, hr_bool rule Id_1); + + Git Branch: cr-596. + Goal: Verify boolean constants work as errorcode/errorlevel in hierarchical + rulesets, including AST round-trip. + """ + code = "GH_598_1" + number_inputs = 1 + references_names = ["1"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_598_1_roundtrip(self): + """ + AST round-trip for check_hierarchy with boolean errorcode/errorlevel. + + Git Branch: cr-596. + Goal: Verify that rendering the AST back to VTL and re-parsing produces the + same result when boolean constants are used in errorcode/errorlevel. + """ + code = "GH_598_1" + number_inputs = 1 + references_names = ["1"] + + text = self.LoadVTL(code) + rendered = ASTString().render(create_ast(text)) + self.BaseTest( + code=code, number_inputs=number_inputs, references_names=references_names, text=rendered + ) + class HierarchicalRollUpOperatorsTest(HierarchicalHelper): """ diff --git a/tests/Validation/data/DataSet/input/GH_598_3-1.csv b/tests/Validation/data/DataSet/input/GH_598_3-1.csv new file mode 100644 index 000000000..c93f7a146 --- /dev/null +++ b/tests/Validation/data/DataSet/input/GH_598_3-1.csv @@ -0,0 +1,5 @@ +Id_1,Id_2,Me_1 +A,1,10 +A,2,-5 +B,1,100 +B,2,30 diff --git a/tests/Validation/data/DataSet/output/GH_598_3-1.csv b/tests/Validation/data/DataSet/output/GH_598_3-1.csv new file mode 100644 index 000000000..006ac0b2b --- /dev/null +++ b/tests/Validation/data/DataSet/output/GH_598_3-1.csv @@ -0,0 +1,2 @@ +Id_1,Id_2,bool_var,imbalance,errorcode,errorlevel +A,2,False,,True,False diff --git a/tests/Validation/data/DataStructure/input/GH_598_3-1.json b/tests/Validation/data/DataStructure/input/GH_598_3-1.json new file mode 100644 index 000000000..a20b58429 --- /dev/null +++ b/tests/Validation/data/DataStructure/input/GH_598_3-1.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Integer", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Validation/data/DataStructure/output/GH_598_3-1.json b/tests/Validation/data/DataStructure/output/GH_598_3-1.json new file mode 100644 index 000000000..911194762 --- /dev/null +++ b/tests/Validation/data/DataStructure/output/GH_598_3-1.json @@ -0,0 +1,45 @@ +{ + "datasets": [ + { + "name": "DS_r", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Integer", + "nullable": false + }, + { + "name": "bool_var", + "role": "Measure", + "type": "Boolean", + "nullable": true + }, + { + "name": "imbalance", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "errorcode", + "role": "Measure", + "type": "String", + "nullable": true + }, + { + "name": "errorlevel", + "role": "Measure", + "type": "Boolean", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Validation/data/vtl/GH_598_3.vtl b/tests/Validation/data/vtl/GH_598_3.vtl new file mode 100644 index 000000000..86b6d3c64 --- /dev/null +++ b/tests/Validation/data/vtl/GH_598_3.vtl @@ -0,0 +1 @@ +DS_r <- check(DS_1#Me_1 > 0 errorcode true errorlevel false invalid); diff --git a/tests/Validation/test_validation.py b/tests/Validation/test_validation.py index a69a4dae9..0a8e751b7 100644 --- a/tests/Validation/test_validation.py +++ b/tests/Validation/test_validation.py @@ -1,6 +1,8 @@ from pathlib import Path from tests.Helper import TestHelper +from vtlengine.API import create_ast +from vtlengine.AST.ASTString import ASTString class ValidationHelper(TestHelper): @@ -451,3 +453,39 @@ def test_GH_427_2(self): references_names = ["1"] self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_598_3(self): + """ + Check with boolean errorcode and errorlevel constants. + Dataset --> Dataset + Status: OK + + DS_r <- check(DS_1#Me_1 > 0 errorcode true errorlevel false invalid); + + Git Branch: cr-596. + Goal: Verify boolean constants work as errorcode/errorlevel in the check + operator, including AST round-trip. + """ + code = "GH_598_3" + number_inputs = 1 + references_names = ["1"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_598_3_roundtrip(self): + """ + AST round-trip for check with boolean errorcode/errorlevel. + + Git Branch: cr-596. + Goal: Verify that rendering the AST back to VTL and re-parsing produces the + same result when boolean constants are used in errorcode/errorlevel. + """ + code = "GH_598_3" + number_inputs = 1 + references_names = ["1"] + + text = self.LoadVTL(code) + rendered = ASTString().render(create_ast(text)) + self.BaseTest( + code=code, number_inputs=number_inputs, references_names=references_names, text=rendered + )