From ca1495a0f0dcd6cd6b146f6bb66b68444526aedf Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Wed, 16 Aug 2023 19:35:51 -0500 Subject: [PATCH 1/7] handle_strict_undefined --- salt/utils/jinja.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index f802156ddb8c..592a4e54ce86 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -734,6 +734,16 @@ def show_full_context(ctx): ) +def _handle_strict_undefined(function): + @wraps(function) + def __handle_strict_undefined(value, *args, **kwargs): + if isinstance(value, jinja2.StrictUndefined): + return value + return function(value, *args, **kwargs) + + return __handle_strict_undefined + + class SerializerExtension(Extension): ''' Yaml and Json manipulation. @@ -956,13 +966,15 @@ def __init__(self, environment): "load_json": self.load_json, "load_text": self.load_text, "dict_to_sls_yaml_params": self.dict_to_sls_yaml_params, - "combinations": itertools.combinations, - "combinations_with_replacement": itertools.combinations_with_replacement, - "compress": itertools.compress, - "permutations": itertools.permutations, - "product": itertools.product, - "zip": zip, - "zip_longest": itertools.zip_longest, + "combinations": _handle_strict_undefined(itertools.combinations), + "combinations_with_replacement": _handle_strict_undefined( + itertools.combinations_with_replacement + ), + "compress": _handle_strict_undefined(itertools.compress), + "permutations": _handle_strict_undefined(itertools.permutations), + "product": _handle_strict_undefined(itertools.product), + "zip": _handle_strict_undefined(zip), + "zip_longest": _handle_strict_undefined(itertools.zip_longest), } ) @@ -993,6 +1005,7 @@ def explore(data): return explore(data) + @_handle_strict_undefined def format_json(self, value, sort_keys=True, indent=None): json_txt = salt.utils.json.dumps( value, sort_keys=sort_keys, indent=indent @@ -1002,6 +1015,7 @@ def format_json(self, value, sort_keys=True, indent=None): except UnicodeDecodeError: return Markup(salt.utils.stringutils.to_unicode(json_txt)) + @_handle_strict_undefined def format_yaml(self, value, flow_style=True): yaml_txt = salt.utils.yaml.safe_dump( value, default_flow_style=flow_style @@ -1013,6 +1027,7 @@ def format_yaml(self, value, flow_style=True): except UnicodeDecodeError: return Markup(salt.utils.stringutils.to_unicode(yaml_txt)) + @_handle_strict_undefined def format_xml(self, value): """Render a formatted multi-line XML string from a complex Python data structure. Supports tag attributes and nested dicts/lists. @@ -1069,9 +1084,11 @@ def recurse_tree(xmliter, element=None): ).toprettyxml(indent=" ") ) + @_handle_strict_undefined def format_python(self, value): return Markup(pprint.pformat(value).strip()) + @_handle_strict_undefined def load_yaml(self, value): if isinstance(value, TemplateModule): value = str(value) @@ -1097,6 +1114,7 @@ def load_yaml(self, value): except AttributeError: raise TemplateRuntimeError(f"Unable to load yaml from {value}") + @_handle_strict_undefined def load_json(self, value): if isinstance(value, TemplateModule): value = str(value) @@ -1105,6 +1123,7 @@ def load_json(self, value): except (ValueError, TypeError, AttributeError): raise TemplateRuntimeError(f"Unable to load json from {value}") + @_handle_strict_undefined def load_text(self, value): if isinstance(value, TemplateModule): value = str(value) @@ -1231,6 +1250,7 @@ def parse_import(self, parser, converter): parser, import_node.template, f"import_{converter}", body, lineno ) + @_handle_strict_undefined def dict_to_sls_yaml_params(self, value, flow_style=False): """ .. versionadded:: 3005 From be31f4b7a35fc55f6c4ef10c9624efb2e604790b Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Wed, 16 Aug 2023 20:30:59 -0500 Subject: [PATCH 2/7] fix dec --- salt/utils/jinja.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 592a4e54ce86..b82d8e339037 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -1007,6 +1007,8 @@ def explore(data): @_handle_strict_undefined def format_json(self, value, sort_keys=True, indent=None): + if isinstance(value, jinja2.StrictUndefined): + return value json_txt = salt.utils.json.dumps( value, sort_keys=sort_keys, indent=indent ).strip() @@ -1015,8 +1017,9 @@ def format_json(self, value, sort_keys=True, indent=None): except UnicodeDecodeError: return Markup(salt.utils.stringutils.to_unicode(json_txt)) - @_handle_strict_undefined def format_yaml(self, value, flow_style=True): + if isinstance(value, jinja2.StrictUndefined): + return value yaml_txt = salt.utils.yaml.safe_dump( value, default_flow_style=flow_style ).strip() @@ -1036,6 +1039,8 @@ def format_xml(self, value): :returns: Formatted XML string rendered with newlines and indentation :rtype: str """ + if isinstance(value, jinja2.StrictUndefined): + return value def normalize_iter(value): if isinstance(value, (list, tuple)): @@ -1084,12 +1089,14 @@ def recurse_tree(xmliter, element=None): ).toprettyxml(indent=" ") ) - @_handle_strict_undefined def format_python(self, value): + if isinstance(value, jinja2.StrictUndefined): + return value return Markup(pprint.pformat(value).strip()) - @_handle_strict_undefined def load_yaml(self, value): + if isinstance(value, jinja2.StrictUndefined): + return value if isinstance(value, TemplateModule): value = str(value) try: @@ -1114,8 +1121,9 @@ def load_yaml(self, value): except AttributeError: raise TemplateRuntimeError(f"Unable to load yaml from {value}") - @_handle_strict_undefined def load_json(self, value): + if isinstance(value, jinja2.StrictUndefined): + return value if isinstance(value, TemplateModule): value = str(value) try: @@ -1123,8 +1131,9 @@ def load_json(self, value): except (ValueError, TypeError, AttributeError): raise TemplateRuntimeError(f"Unable to load json from {value}") - @_handle_strict_undefined def load_text(self, value): + if isinstance(value, jinja2.StrictUndefined): + return value if isinstance(value, TemplateModule): value = str(value) @@ -1250,7 +1259,6 @@ def parse_import(self, parser, converter): parser, import_node.template, f"import_{converter}", body, lineno ) - @_handle_strict_undefined def dict_to_sls_yaml_params(self, value, flow_style=False): """ .. versionadded:: 3005 @@ -1267,6 +1275,8 @@ def dict_to_sls_yaml_params(self, value, flow_style=False): :returns: Formatted SLS YAML string rendered with newlines and indentation """ + if isinstance(value, jinja2.StrictUndefined): + return value return self.format_yaml( [{key: val} for key, val in value.items()], flow_style=flow_style ) From 416c96843d4bbdac94f50671e04b43989963852a Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Tue, 5 Sep 2023 12:23:43 -0500 Subject: [PATCH 3/7] check for nested StrictUndefined --- salt/utils/jinja.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index b82d8e339037..06204a69d60b 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -11,7 +11,7 @@ import time import uuid import warnings -from collections.abc import Hashable +from collections.abc import Hashable, Iterable, Mapping from functools import wraps from xml.dom import minidom from xml.etree.ElementTree import Element, SubElement, tostring @@ -734,10 +734,26 @@ def show_full_context(ctx): ) +def _has_strict_undefined(value): + if isinstance(value, jinja2.StrictUndefined): + return True + elif isinstance(value, Mapping): + for key, item in value.items(): + if _handle_strict_undefined(key): + return True + if _handle_strict_undefined(item): + return True + elif isinstance(value, Iterable): + for item in value: + if _handle_strict_undefined(item): + return True + return False + + def _handle_strict_undefined(function): @wraps(function) def __handle_strict_undefined(value, *args, **kwargs): - if isinstance(value, jinja2.StrictUndefined): + if _has_strict_undefined(value): return value return function(value, *args, **kwargs) @@ -1007,7 +1023,7 @@ def explore(data): @_handle_strict_undefined def format_json(self, value, sort_keys=True, indent=None): - if isinstance(value, jinja2.StrictUndefined): + if _has_strict_undefined(value): return value json_txt = salt.utils.json.dumps( value, sort_keys=sort_keys, indent=indent @@ -1018,7 +1034,7 @@ def format_json(self, value, sort_keys=True, indent=None): return Markup(salt.utils.stringutils.to_unicode(json_txt)) def format_yaml(self, value, flow_style=True): - if isinstance(value, jinja2.StrictUndefined): + if _has_strict_undefined(value): return value yaml_txt = salt.utils.yaml.safe_dump( value, default_flow_style=flow_style @@ -1039,7 +1055,7 @@ def format_xml(self, value): :returns: Formatted XML string rendered with newlines and indentation :rtype: str """ - if isinstance(value, jinja2.StrictUndefined): + if _has_strict_undefined(value): return value def normalize_iter(value): @@ -1090,12 +1106,12 @@ def recurse_tree(xmliter, element=None): ) def format_python(self, value): - if isinstance(value, jinja2.StrictUndefined): + if _has_strict_undefined(value): return value return Markup(pprint.pformat(value).strip()) def load_yaml(self, value): - if isinstance(value, jinja2.StrictUndefined): + if _has_strict_undefined(value): return value if isinstance(value, TemplateModule): value = str(value) @@ -1122,7 +1138,7 @@ def load_yaml(self, value): raise TemplateRuntimeError(f"Unable to load yaml from {value}") def load_json(self, value): - if isinstance(value, jinja2.StrictUndefined): + if _has_strict_undefined(value): return value if isinstance(value, TemplateModule): value = str(value) @@ -1132,7 +1148,7 @@ def load_json(self, value): raise TemplateRuntimeError(f"Unable to load json from {value}") def load_text(self, value): - if isinstance(value, jinja2.StrictUndefined): + if _has_strict_undefined(value): return value if isinstance(value, TemplateModule): value = str(value) @@ -1275,7 +1291,7 @@ def dict_to_sls_yaml_params(self, value, flow_style=False): :returns: Formatted SLS YAML string rendered with newlines and indentation """ - if isinstance(value, jinja2.StrictUndefined): + if _has_strict_undefined(value): return value return self.format_yaml( [{key: val} for key, val in value.items()], flow_style=flow_style From d491aeb0da1979b533e9d9f1bc6e9f447933e419 Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Tue, 5 Sep 2023 13:38:18 -0500 Subject: [PATCH 4/7] tests --- salt/utils/jinja.py | 11 +- .../unit/utils/jinja/test_strict_undefined.py | 116 ++++++++++++++++++ 2 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 tests/pytests/unit/utils/jinja/test_strict_undefined.py diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 06204a69d60b..92467102b160 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -11,7 +11,7 @@ import time import uuid import warnings -from collections.abc import Hashable, Iterable, Mapping +from collections.abc import Hashable, Mapping, Sequence from functools import wraps from xml.dom import minidom from xml.etree.ElementTree import Element, SubElement, tostring @@ -739,13 +739,14 @@ def _has_strict_undefined(value): return True elif isinstance(value, Mapping): for key, item in value.items(): - if _handle_strict_undefined(key): + # StrictUndefined cant be a key in dict, but still check for other mapping types + if _has_strict_undefined(key): return True - if _handle_strict_undefined(item): + if _has_strict_undefined(item): return True - elif isinstance(value, Iterable): + elif isinstance(value, Sequence) and not isinstance(value, str): for item in value: - if _handle_strict_undefined(item): + if _has_strict_undefined(item): return True return False diff --git a/tests/pytests/unit/utils/jinja/test_strict_undefined.py b/tests/pytests/unit/utils/jinja/test_strict_undefined.py new file mode 100644 index 000000000000..01ba74464a26 --- /dev/null +++ b/tests/pytests/unit/utils/jinja/test_strict_undefined.py @@ -0,0 +1,116 @@ +import math + +from jinja2 import StrictUndefined + +from salt.utils import jinja + + +def test_has_undefined(): + assert jinja._has_strict_undefined(StrictUndefined()) is True + + +def test_has_none(): + assert jinja._has_strict_undefined(None) is False + + +def test_has_bool(): + assert jinja._has_strict_undefined(True) is False + assert jinja._has_strict_undefined(False) is False + + +def test_has_int(): + for i in range(-300, 300): + assert jinja._has_strict_undefined(i) is False + assert jinja._has_strict_undefined(8231940728139704) is False + assert jinja._has_strict_undefined(-8231940728139704) is False + + +def test_has_float(): + assert jinja._has_strict_undefined(0.0) is False + assert jinja._has_strict_undefined(-0.0000000000001324) is False + assert jinja._has_strict_undefined(451452.13414) is False + assert jinja._has_strict_undefined(math.inf) is False + assert jinja._has_strict_undefined(math.nan) is False + + +def test_has_str(): + assert jinja._has_strict_undefined("") is False + assert jinja._has_strict_undefined(" ") is False + assert jinja._has_strict_undefined("\0") is False + assert jinja._has_strict_undefined("salt salt salt") is False + assert ( + jinja._has_strict_undefined('assert jinja._has_strict_undefined("") is False') + is False + ) + + +def test_hash_mapping(): + assert jinja._has_strict_undefined({}) is False + assert jinja._has_strict_undefined({True: False, False: True}) is False + assert ( + jinja._has_strict_undefined({True: False, None: True, 88: 98, "salt": 300}) + is False + ) + assert jinja._has_strict_undefined({True: StrictUndefined()}) is True + assert jinja._has_strict_undefined({True: {True: StrictUndefined()}}) is True + assert ( + jinja._has_strict_undefined( + {True: False, None: True, 88: 98, "salt": StrictUndefined()} + ) + is True + ) + + +def test_has_sequence(): + assert jinja._has_strict_undefined(()) is False + assert jinja._has_strict_undefined([]) is False + assert jinja._has_strict_undefined([None, 1, 2, 3]) is False + assert jinja._has_strict_undefined([None, 1, 2, [(None, "str")]]) is False + assert jinja._has_strict_undefined({None, 1, 2, (None, "str")}) is False + assert jinja._has_strict_undefined((StrictUndefined(),)) is True + assert ( + jinja._has_strict_undefined([None, 1, 2, [(None, StrictUndefined())]]) is True + ) + + +def test_has_iter(): + # We should not be running iters make sure iters are not called + assert jinja._has_strict_undefined(iter([])) is False + assert jinja._has_strict_undefined(iter({1: 1})) is False + assert ( + jinja._has_strict_undefined(iter([None, "\0", StrictUndefined(), False])) + is False + ) + assert jinja._has_strict_undefined(iter({1: StrictUndefined()})) is False + + +def test_full(): + assert ( + jinja._has_strict_undefined( + {1: 45, None: (0, 1, [[{"": {"": (3.2, 34.2)}}]], False)} + ) + is False + ) + assert ( + jinja._has_strict_undefined( + {1: 45, None: (0, 1, [[{"": {"": (StrictUndefined(), 34.2)}}]], False)} + ) + is True + ) + + +@jinja._handle_strict_undefined +def _handle_test_helper(value, k=34, *args, **kwargs): + return None + + +def test_handle_strict_undefined(): + assert _handle_test_helper(False) is None + assert _handle_test_helper((1, 2, 3)) is None + assert _handle_test_helper({1, 2, 3}) is None + assert _handle_test_helper([1, 2, 3, {"": None, "str": 0.0}]) is None + # Note StrictUndefined overrides __eq__ + assert isinstance(_handle_test_helper(StrictUndefined()), StrictUndefined) + assert isinstance( + _handle_test_helper([1, 2, 3, {"": None, "str": StrictUndefined()}]), list + ) From 930d587a0fb12f0422a6887ce57fbdc315590d61 Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Tue, 5 Sep 2023 13:42:41 -0500 Subject: [PATCH 5/7] remove dec --- salt/utils/jinja.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 92467102b160..2aeafa1cbe8c 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -1047,7 +1047,6 @@ def format_yaml(self, value, flow_style=True): except UnicodeDecodeError: return Markup(salt.utils.stringutils.to_unicode(yaml_txt)) - @_handle_strict_undefined def format_xml(self, value): """Render a formatted multi-line XML string from a complex Python data structure. Supports tag attributes and nested dicts/lists. From f240eb3b2345bdb554f61660e9d150f4c547d6d6 Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Wed, 6 Sep 2023 23:54:47 -0500 Subject: [PATCH 6/7] wrtie tests and catch nested errors --- changelog/64915.fixed.md | 1 + salt/utils/jinja.py | 68 +++-- .../utils/jinja/test_jinja_custom_filters.py | 267 ++++++++++++++++++ .../unit/utils/jinja/test_strict_undefined.py | 116 -------- 4 files changed, 308 insertions(+), 144 deletions(-) create mode 100644 changelog/64915.fixed.md create mode 100644 tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py delete mode 100644 tests/pytests/unit/utils/jinja/test_strict_undefined.py diff --git a/changelog/64915.fixed.md b/changelog/64915.fixed.md new file mode 100644 index 000000000000..44a599c7eea7 --- /dev/null +++ b/changelog/64915.fixed.md @@ -0,0 +1 @@ +Catch StrictUndefined in salt jinja custom filters. diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 2aeafa1cbe8c..9dc463f5384c 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -734,33 +734,54 @@ def show_full_context(ctx): ) -def _has_strict_undefined(value): +def __get_strict_undefined(value, ids): + if id(value) in ids: + return [] + ids.add(id(value)) + undefined = [] if isinstance(value, jinja2.StrictUndefined): - return True + undefined.append(value) elif isinstance(value, Mapping): for key, item in value.items(): # StrictUndefined cant be a key in dict, but still check for other mapping types - if _has_strict_undefined(key): - return True - if _has_strict_undefined(item): - return True + undefined.extend(__get_strict_undefined(key, ids)) + undefined.extend(__get_strict_undefined(item, ids)) elif isinstance(value, Sequence) and not isinstance(value, str): for item in value: - if _has_strict_undefined(item): - return True - return False + undefined.extend(__get_strict_undefined(item, ids)) + return undefined + + +def _get_strict_undefined(value): + return tuple(__get_strict_undefined(value, set())) + + +def _join_strict_undefined(undefined): + return jinja2.StrictUndefined("\n".join(u._undefined_message for u in undefined)) def _handle_strict_undefined(function): @wraps(function) def __handle_strict_undefined(value, *args, **kwargs): - if _has_strict_undefined(value): - return value + undefined = _get_strict_undefined(value) + if undefined: + return _join_strict_undefined(undefined) return function(value, *args, **kwargs) return __handle_strict_undefined +def _handle_method_strict_undefined(function): + @wraps(function) + def __handle_method_strict_undefined(self, value, *args, **kwargs): + undefined = _get_strict_undefined(value) + if undefined: + return _join_strict_undefined(undefined) + return function(self, value, *args, **kwargs) + + return __handle_method_strict_undefined + + class SerializerExtension(Extension): ''' Yaml and Json manipulation. @@ -1022,10 +1043,8 @@ def explore(data): return explore(data) - @_handle_strict_undefined + @_handle_method_strict_undefined def format_json(self, value, sort_keys=True, indent=None): - if _has_strict_undefined(value): - return value json_txt = salt.utils.json.dumps( value, sort_keys=sort_keys, indent=indent ).strip() @@ -1034,9 +1053,8 @@ def format_json(self, value, sort_keys=True, indent=None): except UnicodeDecodeError: return Markup(salt.utils.stringutils.to_unicode(json_txt)) + @_handle_method_strict_undefined def format_yaml(self, value, flow_style=True): - if _has_strict_undefined(value): - return value yaml_txt = salt.utils.yaml.safe_dump( value, default_flow_style=flow_style ).strip() @@ -1047,6 +1065,7 @@ def format_yaml(self, value, flow_style=True): except UnicodeDecodeError: return Markup(salt.utils.stringutils.to_unicode(yaml_txt)) + @_handle_method_strict_undefined def format_xml(self, value): """Render a formatted multi-line XML string from a complex Python data structure. Supports tag attributes and nested dicts/lists. @@ -1055,8 +1074,6 @@ def format_xml(self, value): :returns: Formatted XML string rendered with newlines and indentation :rtype: str """ - if _has_strict_undefined(value): - return value def normalize_iter(value): if isinstance(value, (list, tuple)): @@ -1105,14 +1122,12 @@ def recurse_tree(xmliter, element=None): ).toprettyxml(indent=" ") ) + @_handle_method_strict_undefined def format_python(self, value): - if _has_strict_undefined(value): - return value return Markup(pprint.pformat(value).strip()) + @_handle_method_strict_undefined def load_yaml(self, value): - if _has_strict_undefined(value): - return value if isinstance(value, TemplateModule): value = str(value) try: @@ -1137,9 +1152,8 @@ def load_yaml(self, value): except AttributeError: raise TemplateRuntimeError(f"Unable to load yaml from {value}") + @_handle_method_strict_undefined def load_json(self, value): - if _has_strict_undefined(value): - return value if isinstance(value, TemplateModule): value = str(value) try: @@ -1147,9 +1161,8 @@ def load_json(self, value): except (ValueError, TypeError, AttributeError): raise TemplateRuntimeError(f"Unable to load json from {value}") + @_handle_method_strict_undefined def load_text(self, value): - if _has_strict_undefined(value): - return value if isinstance(value, TemplateModule): value = str(value) @@ -1275,6 +1288,7 @@ def parse_import(self, parser, converter): parser, import_node.template, f"import_{converter}", body, lineno ) + @_handle_method_strict_undefined def dict_to_sls_yaml_params(self, value, flow_style=False): """ .. versionadded:: 3005 @@ -1291,8 +1305,6 @@ def dict_to_sls_yaml_params(self, value, flow_style=False): :returns: Formatted SLS YAML string rendered with newlines and indentation """ - if _has_strict_undefined(value): - return value return self.format_yaml( [{key: val} for key, val in value.items()], flow_style=flow_style ) diff --git a/tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py b/tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py new file mode 100644 index 000000000000..e6c4f67be371 --- /dev/null +++ b/tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py @@ -0,0 +1,267 @@ +import math + +import jinja2 +import pytest +from jinja2 import StrictUndefined + +from salt.utils import jinja + + +def test_get_undefined(): + assert len(jinja._get_strict_undefined(StrictUndefined())) == 1 + + +def test_get_none(): + assert len(jinja._get_strict_undefined(None)) == 0 + + +def test_get_bool(): + assert len(jinja._get_strict_undefined(True)) == 0 + assert len(jinja._get_strict_undefined(False)) == 0 + + +def test_get_int(): + for i in range(-300, 300): + assert len(jinja._get_strict_undefined(i)) == 0 + assert len(jinja._get_strict_undefined(8231940728139704)) == 0 + assert len(jinja._get_strict_undefined(-8231940728139704)) == 0 + + +def test_get_float(): + assert bool(jinja._get_strict_undefined(0.0)) == 0 + assert bool(jinja._get_strict_undefined(-0.0000000000001324)) == 0 + assert bool(jinja._get_strict_undefined(451452.13414)) == 0 + assert bool(jinja._get_strict_undefined(math.inf)) == 0 + assert bool(jinja._get_strict_undefined(math.nan)) == 0 + + +def test_get_str(): + assert len(jinja._get_strict_undefined("")) == 0 + assert len(jinja._get_strict_undefined(" ")) == 0 + assert len(jinja._get_strict_undefined("\0")) == 0 + assert len(jinja._get_strict_undefined("salt salt salt")) == 0 + assert ( + len( + jinja._get_strict_undefined( + 'assert jinja._has_strict_undefined("") is False' + ) + ) + == 0 + ) + + +def test_get_mapping(): + assert len(jinja._get_strict_undefined({})) == 0 + assert len(jinja._get_strict_undefined({True: False, False: True})) == 0 + assert ( + len(jinja._get_strict_undefined({True: False, None: True, 88: 98, "salt": 300})) + == 0 + ) + assert len(jinja._get_strict_undefined({True: StrictUndefined()})) == 1 + assert len(jinja._get_strict_undefined({True: {True: StrictUndefined()}})) == 1 + assert ( + len( + jinja._get_strict_undefined( + {True: False, None: True, 88: 98, "salt": StrictUndefined()} + ) + ) + == 1 + ) + + +def test_get_sequence(): + assert len(jinja._get_strict_undefined(())) == 0 + assert len(jinja._get_strict_undefined([])) == 0 + assert len(jinja._get_strict_undefined([None, 1, 2, 3])) == 0 + assert len(jinja._get_strict_undefined([None, 1, 2, [(None, "str")]])) == 0 + assert len(jinja._get_strict_undefined({None, 1, 2, (None, "str")})) == 0 + assert len(jinja._get_strict_undefined((StrictUndefined(),))) == 1 + assert ( + len(jinja._get_strict_undefined([None, 1, 2, [(None, StrictUndefined())]])) == 1 + ) + + +def test_get_iter(): + # We should not be running iters make sure iters are not called + assert len(jinja._get_strict_undefined(iter([]))) == 0 + assert len(jinja._get_strict_undefined(iter({1: 1}))) == 0 + assert ( + len(jinja._get_strict_undefined(iter([None, "\0", StrictUndefined(), False]))) + == 0 + ) + assert len(jinja._get_strict_undefined(iter({1: StrictUndefined()}))) == 0 + + +def test_full(): + assert ( + len( + jinja._get_strict_undefined( + {1: 45, None: (0, 1, [[{"": {"": (3.2, 34.2)}}]], False)} + ) + ) + == 0 + ) + assert ( + len( + jinja._get_strict_undefined( + {1: 45, None: (0, 1, [[{"": {"": (StrictUndefined(), 34.2)}}]], False)} + ) + ) + == 1 + ) + assert ( + len( + jinja._get_strict_undefined( + { + 1: 45, + None: ( + 0, + 1, + [ + [ + { + "": {"": (StrictUndefined(), 34.2)}, + 45: StrictUndefined(), + } + ] + ], + False, + ), + } + ) + ) + == 2 + ) + assert ( + len( + jinja._get_strict_undefined( + { + 1: 45, + None: ( + 0, + 1, + [ + StrictUndefined(), + [ + { + "": {"": (StrictUndefined(), 34.2)}, + 45: StrictUndefined(), + } + ], + StrictUndefined(), + ], + False, + ), + } + ) + ) + == 4 + ) + + +@jinja._handle_strict_undefined +def _handle_test_helper(value, k=34, *args, **kwargs): + return None + + +def test_handle_strict_undefined(): + assert _handle_test_helper(False) is None + assert _handle_test_helper((1, 2, 3)) is None + assert _handle_test_helper({1, 2, 3}) is None + assert _handle_test_helper([1, 2, 3, {"": None, "str": 0.0}]) is None + assert isinstance(_handle_test_helper(StrictUndefined()), StrictUndefined) + assert isinstance( + _handle_test_helper( + [1, 2, 3, {"": StrictUndefined(), "str": StrictUndefined()}] + ), + StrictUndefined, + ) + + +def _render(yaml): + return ( + jinja2.Environment( + extensions=[jinja.SerializerExtension], undefined=jinja2.StrictUndefined + ) + .from_string(yaml) + .render() + ) + + +def _render_fail(yaml): + with pytest.raises(jinja2.exceptions.UndefinedError): + _render(yaml) + + +YAML_SLS_ERROR = """ +{%- set ports = {'http': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ ports['https'] | yaml }} +""" + + +YAML_SLS = """ +{%- set ports = {'https': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ ports['https'] | yaml }} +""" + + +YAML_SLS_RIGHT = """ + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is 80""" + + +def test_yaml(): + _render_fail(YAML_SLS_ERROR) + assert _render(YAML_SLS) == YAML_SLS_RIGHT + + +JSON_SLS_ERROR = """ +{%- set ports = {'http': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ [ports['https23'], ports['https']] | json }} + - comment2: https port is {{ [666, ports['https2']] | json }} + - comment3: https port is {{ [666, ports['https2']] | json }} +""" + + +JSON_SLS = """ +{%- set ports = {'https': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ [666, ports['https']] | json }} +""" + + +JSON_SLS_RIGHT = """ + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is [666, 80]""" + + +def test_json(): + _render_fail(JSON_SLS_ERROR) + assert _render(JSON_SLS) == JSON_SLS_RIGHT diff --git a/tests/pytests/unit/utils/jinja/test_strict_undefined.py b/tests/pytests/unit/utils/jinja/test_strict_undefined.py deleted file mode 100644 index 01ba74464a26..000000000000 --- a/tests/pytests/unit/utils/jinja/test_strict_undefined.py +++ /dev/null @@ -1,116 +0,0 @@ -import math - -from jinja2 import StrictUndefined - -from salt.utils import jinja - - -def test_has_undefined(): - assert jinja._has_strict_undefined(StrictUndefined()) is True - - -def test_has_none(): - assert jinja._has_strict_undefined(None) is False - - -def test_has_bool(): - assert jinja._has_strict_undefined(True) is False - assert jinja._has_strict_undefined(False) is False - - -def test_has_int(): - for i in range(-300, 300): - assert jinja._has_strict_undefined(i) is False - assert jinja._has_strict_undefined(8231940728139704) is False - assert jinja._has_strict_undefined(-8231940728139704) is False - - -def test_has_float(): - assert jinja._has_strict_undefined(0.0) is False - assert jinja._has_strict_undefined(-0.0000000000001324) is False - assert jinja._has_strict_undefined(451452.13414) is False - assert jinja._has_strict_undefined(math.inf) is False - assert jinja._has_strict_undefined(math.nan) is False - - -def test_has_str(): - assert jinja._has_strict_undefined("") is False - assert jinja._has_strict_undefined(" ") is False - assert jinja._has_strict_undefined("\0") is False - assert jinja._has_strict_undefined("salt salt salt") is False - assert ( - jinja._has_strict_undefined('assert jinja._has_strict_undefined("") is False') - is False - ) - - -def test_hash_mapping(): - assert jinja._has_strict_undefined({}) is False - assert jinja._has_strict_undefined({True: False, False: True}) is False - assert ( - jinja._has_strict_undefined({True: False, None: True, 88: 98, "salt": 300}) - is False - ) - assert jinja._has_strict_undefined({True: StrictUndefined()}) is True - assert jinja._has_strict_undefined({True: {True: StrictUndefined()}}) is True - assert ( - jinja._has_strict_undefined( - {True: False, None: True, 88: 98, "salt": StrictUndefined()} - ) - is True - ) - - -def test_has_sequence(): - assert jinja._has_strict_undefined(()) is False - assert jinja._has_strict_undefined([]) is False - assert jinja._has_strict_undefined([None, 1, 2, 3]) is False - assert jinja._has_strict_undefined([None, 1, 2, [(None, "str")]]) is False - assert jinja._has_strict_undefined({None, 1, 2, (None, "str")}) is False - assert jinja._has_strict_undefined((StrictUndefined(),)) is True - assert ( - jinja._has_strict_undefined([None, 1, 2, [(None, StrictUndefined())]]) is True - ) - - -def test_has_iter(): - # We should not be running iters make sure iters are not called - assert jinja._has_strict_undefined(iter([])) is False - assert jinja._has_strict_undefined(iter({1: 1})) is False - assert ( - jinja._has_strict_undefined(iter([None, "\0", StrictUndefined(), False])) - is False - ) - assert jinja._has_strict_undefined(iter({1: StrictUndefined()})) is False - - -def test_full(): - assert ( - jinja._has_strict_undefined( - {1: 45, None: (0, 1, [[{"": {"": (3.2, 34.2)}}]], False)} - ) - is False - ) - assert ( - jinja._has_strict_undefined( - {1: 45, None: (0, 1, [[{"": {"": (StrictUndefined(), 34.2)}}]], False)} - ) - is True - ) - - -@jinja._handle_strict_undefined -def _handle_test_helper(value, k=34, *args, **kwargs): - return None - - -def test_handle_strict_undefined(): - assert _handle_test_helper(False) is None - assert _handle_test_helper((1, 2, 3)) is None - assert _handle_test_helper({1, 2, 3}) is None - assert _handle_test_helper([1, 2, 3, {"": None, "str": 0.0}]) is None - # Note StrictUndefined overrides __eq__ - assert isinstance(_handle_test_helper(StrictUndefined()), StrictUndefined) - assert isinstance( - _handle_test_helper([1, 2, 3, {"": None, "str": StrictUndefined()}]), list - ) From f9142857a068df86e29fa02d78ed0199145c46d4 Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Wed, 18 Oct 2023 11:31:52 -0500 Subject: [PATCH 7/7] add python tests --- .../utils/jinja/test_jinja_custom_filters.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py b/tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py index e6c4f67be371..38abdfd53b2f 100644 --- a/tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py +++ b/tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py @@ -265,3 +265,40 @@ def test_yaml(): def test_json(): _render_fail(JSON_SLS_ERROR) assert _render(JSON_SLS) == JSON_SLS_RIGHT + +PYTHON_SLS_ERROR = """ +{%- set ports = {'http': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ [ports['https23'], ports['https']] | python }} + - comment2: https port is {{ [666 + 1, ports['https2']] | python }} + - comment3: https port is {{ [666 + 1, ports['https2']] | python }} +""" + + +PYTHON_SLS = """ +{%- set ports = {'https': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ [666 + 1, ports['https']] | python }} +""" + + +PYTHON_SLS_RIGHT = """ + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is [667, 80]""" + + +def test_python(): + _render_fail(PYTHON_SLS_ERROR) + assert _render(PYTHON_SLS) == PYTHON_SLS_RIGHT