diff --git a/changelog/64915.fixed.md b/changelog/64915.fixed.md new file mode 100644 index 000000000000..44a599c7eea7 --- /dev/null +++ b/changelog/64915.fixed.md @@ -0,0 +1 @@ +Catch StrictUndefined in salt jinja custom filters. diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index f802156ddb8c..9dc463f5384c 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -11,7 +11,7 @@ import time import uuid import warnings -from collections.abc import Hashable +from collections.abc import Hashable, Mapping, Sequence from functools import wraps from xml.dom import minidom from xml.etree.ElementTree import Element, SubElement, tostring @@ -734,6 +734,54 @@ def show_full_context(ctx): ) +def __get_strict_undefined(value, ids): + if id(value) in ids: + return [] + ids.add(id(value)) + undefined = [] + if isinstance(value, jinja2.StrictUndefined): + undefined.append(value) + elif isinstance(value, Mapping): + for key, item in value.items(): + # StrictUndefined cant be a key in dict, but still check for other mapping types + undefined.extend(__get_strict_undefined(key, ids)) + undefined.extend(__get_strict_undefined(item, ids)) + elif isinstance(value, Sequence) and not isinstance(value, str): + for item in value: + undefined.extend(__get_strict_undefined(item, ids)) + return undefined + + +def _get_strict_undefined(value): + return tuple(__get_strict_undefined(value, set())) + + +def _join_strict_undefined(undefined): + return jinja2.StrictUndefined("\n".join(u._undefined_message for u in undefined)) + + +def _handle_strict_undefined(function): + @wraps(function) + def __handle_strict_undefined(value, *args, **kwargs): + undefined = _get_strict_undefined(value) + if undefined: + return _join_strict_undefined(undefined) + return function(value, *args, **kwargs) + + return __handle_strict_undefined + + +def _handle_method_strict_undefined(function): + @wraps(function) + def __handle_method_strict_undefined(self, value, *args, **kwargs): + undefined = _get_strict_undefined(value) + if undefined: + return _join_strict_undefined(undefined) + return function(self, value, *args, **kwargs) + + return __handle_method_strict_undefined + + class SerializerExtension(Extension): ''' Yaml and Json manipulation. @@ -956,13 +1004,15 @@ def __init__(self, environment): "load_json": self.load_json, "load_text": self.load_text, "dict_to_sls_yaml_params": self.dict_to_sls_yaml_params, - "combinations": itertools.combinations, - "combinations_with_replacement": itertools.combinations_with_replacement, - "compress": itertools.compress, - "permutations": itertools.permutations, - "product": itertools.product, - "zip": zip, - "zip_longest": itertools.zip_longest, + "combinations": _handle_strict_undefined(itertools.combinations), + "combinations_with_replacement": _handle_strict_undefined( + itertools.combinations_with_replacement + ), + "compress": _handle_strict_undefined(itertools.compress), + "permutations": _handle_strict_undefined(itertools.permutations), + "product": _handle_strict_undefined(itertools.product), + "zip": _handle_strict_undefined(zip), + "zip_longest": _handle_strict_undefined(itertools.zip_longest), } ) @@ -993,6 +1043,7 @@ def explore(data): return explore(data) + @_handle_method_strict_undefined def format_json(self, value, sort_keys=True, indent=None): json_txt = salt.utils.json.dumps( value, sort_keys=sort_keys, indent=indent @@ -1002,6 +1053,7 @@ def format_json(self, value, sort_keys=True, indent=None): except UnicodeDecodeError: return Markup(salt.utils.stringutils.to_unicode(json_txt)) + @_handle_method_strict_undefined def format_yaml(self, value, flow_style=True): yaml_txt = salt.utils.yaml.safe_dump( value, default_flow_style=flow_style @@ -1013,6 +1065,7 @@ def format_yaml(self, value, flow_style=True): except UnicodeDecodeError: return Markup(salt.utils.stringutils.to_unicode(yaml_txt)) + @_handle_method_strict_undefined def format_xml(self, value): """Render a formatted multi-line XML string from a complex Python data structure. Supports tag attributes and nested dicts/lists. @@ -1069,9 +1122,11 @@ def recurse_tree(xmliter, element=None): ).toprettyxml(indent=" ") ) + @_handle_method_strict_undefined def format_python(self, value): return Markup(pprint.pformat(value).strip()) + @_handle_method_strict_undefined def load_yaml(self, value): if isinstance(value, TemplateModule): value = str(value) @@ -1097,6 +1152,7 @@ def load_yaml(self, value): except AttributeError: raise TemplateRuntimeError(f"Unable to load yaml from {value}") + @_handle_method_strict_undefined def load_json(self, value): if isinstance(value, TemplateModule): value = str(value) @@ -1105,6 +1161,7 @@ def load_json(self, value): except (ValueError, TypeError, AttributeError): raise TemplateRuntimeError(f"Unable to load json from {value}") + @_handle_method_strict_undefined def load_text(self, value): if isinstance(value, TemplateModule): value = str(value) @@ -1231,6 +1288,7 @@ def parse_import(self, parser, converter): parser, import_node.template, f"import_{converter}", body, lineno ) + @_handle_method_strict_undefined def dict_to_sls_yaml_params(self, value, flow_style=False): """ .. versionadded:: 3005 diff --git a/tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py b/tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py new file mode 100644 index 000000000000..38abdfd53b2f --- /dev/null +++ b/tests/pytests/unit/utils/jinja/test_jinja_custom_filters.py @@ -0,0 +1,304 @@ +import math + +import jinja2 +import pytest +from jinja2 import StrictUndefined + +from salt.utils import jinja + + +def test_get_undefined(): + assert len(jinja._get_strict_undefined(StrictUndefined())) == 1 + + +def test_get_none(): + assert len(jinja._get_strict_undefined(None)) == 0 + + +def test_get_bool(): + assert len(jinja._get_strict_undefined(True)) == 0 + assert len(jinja._get_strict_undefined(False)) == 0 + + +def test_get_int(): + for i in range(-300, 300): + assert len(jinja._get_strict_undefined(i)) == 0 + assert len(jinja._get_strict_undefined(8231940728139704)) == 0 + assert len(jinja._get_strict_undefined(-8231940728139704)) == 0 + + +def test_get_float(): + assert bool(jinja._get_strict_undefined(0.0)) == 0 + assert bool(jinja._get_strict_undefined(-0.0000000000001324)) == 0 + assert bool(jinja._get_strict_undefined(451452.13414)) == 0 + assert bool(jinja._get_strict_undefined(math.inf)) == 0 + assert bool(jinja._get_strict_undefined(math.nan)) == 0 + + +def test_get_str(): + assert len(jinja._get_strict_undefined("")) == 0 + assert len(jinja._get_strict_undefined(" ")) == 0 + assert len(jinja._get_strict_undefined("\0")) == 0 + assert len(jinja._get_strict_undefined("salt salt salt")) == 0 + assert ( + len( + jinja._get_strict_undefined( + 'assert jinja._has_strict_undefined("") is False' + ) + ) + == 0 + ) + + +def test_get_mapping(): + assert len(jinja._get_strict_undefined({})) == 0 + assert len(jinja._get_strict_undefined({True: False, False: True})) == 0 + assert ( + len(jinja._get_strict_undefined({True: False, None: True, 88: 98, "salt": 300})) + == 0 + ) + assert len(jinja._get_strict_undefined({True: StrictUndefined()})) == 1 + assert len(jinja._get_strict_undefined({True: {True: StrictUndefined()}})) == 1 + assert ( + len( + jinja._get_strict_undefined( + {True: False, None: True, 88: 98, "salt": StrictUndefined()} + ) + ) + == 1 + ) + + +def test_get_sequence(): + assert len(jinja._get_strict_undefined(())) == 0 + assert len(jinja._get_strict_undefined([])) == 0 + assert len(jinja._get_strict_undefined([None, 1, 2, 3])) == 0 + assert len(jinja._get_strict_undefined([None, 1, 2, [(None, "str")]])) == 0 + assert len(jinja._get_strict_undefined({None, 1, 2, (None, "str")})) == 0 + assert len(jinja._get_strict_undefined((StrictUndefined(),))) == 1 + assert ( + len(jinja._get_strict_undefined([None, 1, 2, [(None, StrictUndefined())]])) == 1 + ) + + +def test_get_iter(): + # We should not be running iters make sure iters are not called + assert len(jinja._get_strict_undefined(iter([]))) == 0 + assert len(jinja._get_strict_undefined(iter({1: 1}))) == 0 + assert ( + len(jinja._get_strict_undefined(iter([None, "\0", StrictUndefined(), False]))) + == 0 + ) + assert len(jinja._get_strict_undefined(iter({1: StrictUndefined()}))) == 0 + + +def test_full(): + assert ( + len( + jinja._get_strict_undefined( + {1: 45, None: (0, 1, [[{"": {"": (3.2, 34.2)}}]], False)} + ) + ) + == 0 + ) + assert ( + len( + jinja._get_strict_undefined( + {1: 45, None: (0, 1, [[{"": {"": (StrictUndefined(), 34.2)}}]], False)} + ) + ) + == 1 + ) + assert ( + len( + jinja._get_strict_undefined( + { + 1: 45, + None: ( + 0, + 1, + [ + [ + { + "": {"": (StrictUndefined(), 34.2)}, + 45: StrictUndefined(), + } + ] + ], + False, + ), + } + ) + ) + == 2 + ) + assert ( + len( + jinja._get_strict_undefined( + { + 1: 45, + None: ( + 0, + 1, + [ + StrictUndefined(), + [ + { + "": {"": (StrictUndefined(), 34.2)}, + 45: StrictUndefined(), + } + ], + StrictUndefined(), + ], + False, + ), + } + ) + ) + == 4 + ) + + +@jinja._handle_strict_undefined +def _handle_test_helper(value, k=34, *args, **kwargs): + return None + + +def test_handle_strict_undefined(): + assert _handle_test_helper(False) is None + assert _handle_test_helper((1, 2, 3)) is None + assert _handle_test_helper({1, 2, 3}) is None + assert _handle_test_helper([1, 2, 3, {"": None, "str": 0.0}]) is None + assert isinstance(_handle_test_helper(StrictUndefined()), StrictUndefined) + assert isinstance( + _handle_test_helper( + [1, 2, 3, {"": StrictUndefined(), "str": StrictUndefined()}] + ), + StrictUndefined, + ) + + +def _render(yaml): + return ( + jinja2.Environment( + extensions=[jinja.SerializerExtension], undefined=jinja2.StrictUndefined + ) + .from_string(yaml) + .render() + ) + + +def _render_fail(yaml): + with pytest.raises(jinja2.exceptions.UndefinedError): + _render(yaml) + + +YAML_SLS_ERROR = """ +{%- set ports = {'http': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ ports['https'] | yaml }} +""" + + +YAML_SLS = """ +{%- set ports = {'https': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ ports['https'] | yaml }} +""" + + +YAML_SLS_RIGHT = """ + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is 80""" + + +def test_yaml(): + _render_fail(YAML_SLS_ERROR) + assert _render(YAML_SLS) == YAML_SLS_RIGHT + + +JSON_SLS_ERROR = """ +{%- set ports = {'http': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ [ports['https23'], ports['https']] | json }} + - comment2: https port is {{ [666, ports['https2']] | json }} + - comment3: https port is {{ [666, ports['https2']] | json }} +""" + + +JSON_SLS = """ +{%- set ports = {'https': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ [666, ports['https']] | json }} +""" + + +JSON_SLS_RIGHT = """ + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is [666, 80]""" + + +def test_json(): + _render_fail(JSON_SLS_ERROR) + assert _render(JSON_SLS) == JSON_SLS_RIGHT + +PYTHON_SLS_ERROR = """ +{%- set ports = {'http': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ [ports['https23'], ports['https']] | python }} + - comment2: https port is {{ [666 + 1, ports['https2']] | python }} + - comment3: https port is {{ [666 + 1, ports['https2']] | python }} +""" + + +PYTHON_SLS = """ +{%- set ports = {'https': 80} %} + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is {{ [666 + 1, ports['https']] | python }} +""" + + +PYTHON_SLS_RIGHT = """ + +huh: + test.configurable_test_state: + - changes: true + - result: true + - comment: https port is [667, 80]""" + + +def test_python(): + _render_fail(PYTHON_SLS_ERROR) + assert _render(PYTHON_SLS) == PYTHON_SLS_RIGHT