diff --git a/Development.adoc b/Development.adoc index a79c47c..87f1078 100644 --- a/Development.adoc +++ b/Development.adoc @@ -114,8 +114,9 @@ helm template helm \ --set manageClaimsInterval=5 \ --set manageHandlesInterval=5 \ --set operatorDomain.name=poolboy.dev.local \ ---set=image.tagOverride=- \ ---set=image.repository=$(oc get imagestream poolboy -o jsonpath='{.status.tags[?(@.tag=="latest")].items[0].dockerImageReference}') \ +--set resourceHandlerCount=2 \ +--set image.tagOverride=- \ +--set image.repository=$(oc get imagestream poolboy -o jsonpath='{.status.tags[?(@.tag=="latest")].items[0].dockerImageReference}') \ | oc apply -f - -------------------------------------------------------------------------------- diff --git a/operator/poolboy_templating.py b/operator/poolboy_templating.py index a11ebac..87a79ca 100644 --- a/operator/poolboy_templating.py +++ b/operator/poolboy_templating.py @@ -101,20 +101,18 @@ def timedelta_to_str(td:timedelta) -> str: ret += f"{seconds}s" return ret -jinja2envs = { - 'jinja2': jinja2.Environment( - finalize = error_if_undefined, - undefined = jinja2.ChainableUndefined, - ), -} -jinja2envs['jinja2'].filters['bool'] = lambda x: bool(str2bool(x)) if isinstance(x, str) else bool(x) -jinja2envs['jinja2'].filters['json_query'] = lambda x, query: jmespath.search(query, x) -jinja2envs['jinja2'].filters['merge_list_of_dicts'] = lambda a: functools.reduce(lambda d1, d2: {**(d1 or {}), **(d2 or {})}, a) if a else {} -jinja2envs['jinja2'].filters['object'] = lambda x: json.dumps(x) -jinja2envs['jinja2'].filters['parse_time_interval'] = lambda x: timedelta(seconds=pytimeparse.parse(str(x))) -jinja2envs['jinja2'].filters['strgen'] = lambda x: StringGenerator(x).render() -jinja2envs['jinja2'].filters['to_datetime'] = lambda s, f='%Y-%m-%d %H:%M:%S': datetime.strptime(str(s), f) -jinja2envs['jinja2'].filters['to_json'] = lambda x: json.dumps(x) +jinja2env = jinja2.Environment( + finalize = error_if_undefined, + undefined = jinja2.ChainableUndefined, +) +jinja2env.filters['bool'] = lambda x: bool(str2bool(x)) if isinstance(x, str) else bool(x) +jinja2env.filters['json_query'] = lambda x, query: jmespath.search(query, x) +jinja2env.filters['merge_list_of_dicts'] = lambda a: functools.reduce(lambda d1, d2: {**(d1 or {}), **(d2 or {})}, a) if a else {} +jinja2env.filters['object'] = lambda x: json.dumps(x) +jinja2env.filters['parse_time_interval'] = lambda x: timedelta(seconds=pytimeparse.parse(str(x))) +jinja2env.filters['strgen'] = lambda x: StringGenerator(x).render() +jinja2env.filters['to_datetime'] = lambda s, f='%Y-%m-%d %H:%M:%S': datetime.strptime(str(s), f) +jinja2env.filters['to_json'] = lambda x: json.dumps(x) # Regex to detect if it looks like this value should be rendered as a raw type # rather than a string. @@ -146,10 +144,9 @@ def timedelta_to_str(td:timedelta) -> str: # name: alice type_filter_match_re = re.compile(r'^{{(?!.*{{).*\| *(bool|float|int|list|object) *}}$') -def check_condition(condition, template_style='jinja2', variables={}, template_variables={}): +def check_condition(condition, variables={}, template_variables={}): return jinja2process( template="{{ (" + condition + ") | bool}}", - template_style=template_style, variables=variables, template_variables=template_variables, ) @@ -178,7 +175,15 @@ def j2now(utc=False, fmt=None): dt = datetime.now(timezone.utc if utc else None) return dt.strftime(fmt) if fmt else dt -def jinja2process(template, omit=None, template_style='jinja2', variables={}, template_variables={}): +j2template_cache = {} +def j2template_get(template: str): + if template in j2template_cache: + return j2template_cache[template] + j2template = jinja2env.from_string(template) + j2template_cache[template] = j2template + return j2template + +def jinja2process(template, omit=None, variables={}, template_variables={}): variables = copy.copy(variables) variables['datetime'] = datetime variables['now'] = j2now @@ -187,18 +192,19 @@ def jinja2process(template, omit=None, template_style='jinja2', variables={}, te variables['timezone'] = timezone variables['timestamp'] = TimeStamp() variables['__recursion_depth__'] = 0 - jinja2env = jinja2envs.get(template_style) - j2template = jinja2env.from_string(template) + j2template = j2template_get(template) class TemplateVariable(object): '''Variable may contain template string referencing other variables.''' def __init__(self, value): - self.template = value - self.j2template = jinja2env.from_string(value) + self.value = value + self.j2template = j2template_get(value) if '{{' in value else None def get_typed_value(self): - template_out = self.__str__() - return template_output_typing(template_out, self.template) + if self.j2template: + template_out = self.__str__() + return template_output_typing(template_out, self.value) + return self.value def __bool__(self): return str2bool(self.__str__()) @@ -255,6 +261,8 @@ def __repr__(self): return self.__str__() def __str__(self): + if self.j2template is None: + return str(self.value) if variables['__recursion_depth__'] > MAX_RECURSION_DEPTH: raise RuntimeError("Template variable exceeded max recursion") variables['__recursion_depth__'] += 1 @@ -268,7 +276,7 @@ def __str__(self): template_out = j2template.render(variables) return template_output_typing(template_out, template) -def recursive_process_template_strings(template, template_style='jinja2', variables={}, template_variables={}): +def recursive_process_template_strings(template, variables={}, template_variables={}): """Take a template and recursively process template strings within it. The template may be any type. If it is a dictionary or list then all values will be recursively procesed. @@ -280,7 +288,6 @@ def recursive_process_template_strings(template, template_style='jinja2', variab validating parameters, checking heath/readiness, and generating resource definitions. Keyword arguments: - template_style -- Style of templates used. Currently only "jinja2" is supported variables -- simple key/value pair variables template_variables -- variables which may contain template strings and should only come from trusted sources """ @@ -289,26 +296,25 @@ def recursive_process_template_strings(template, template_style='jinja2', variab __recursive_process_template_strings( omit = omit, template = template, - template_style = template_style, variables = variables, template_variables = template_variables, ), omit = omit, ) -def __recursive_process_template_strings(template, omit, template_style, variables, template_variables): +def __recursive_process_template_strings(template, omit, variables, template_variables): if isinstance(template, dict): return { - key: __recursive_process_template_strings(val, omit=omit, template_style=template_style, variables=variables, template_variables=template_variables) + key: __recursive_process_template_strings(val, omit=omit, variables=variables, template_variables=template_variables) for key, val in template.items() } elif isinstance(template, list): return [ - __recursive_process_template_strings(item, omit=omit, template_style=template_style, variables=variables, template_variables=template_variables) + __recursive_process_template_strings(item, omit=omit, variables=variables, template_variables=template_variables) for item in template ] elif isinstance(template, str): - return jinja2process(template, omit=omit, template_style=template_style, variables=variables, template_variables=template_variables) + return jinja2process(template, omit=omit, variables=variables, template_variables=template_variables) else: return template diff --git a/operator/resourceprovider.py b/operator/resourceprovider.py index 1fabb1c..ca8371b 100644 --- a/operator/resourceprovider.py +++ b/operator/resourceprovider.py @@ -68,7 +68,7 @@ def check_wait_for(self, ) return recursive_process_template_strings( - '{{(' + self.wait_for + ')|bool}}', resource_provider.template_style, + '{{(' + self.wait_for + ')|bool}}', variables = vars_, template_variables = {**resource_provider.vars, **resource_handle.vars}, ) @@ -92,7 +92,7 @@ def check_when(self, resource_handle_vars = resource_handle.vars if resource_handle else {} return recursive_process_template_strings( - '{{(' + self.when + ')|bool}}', resource_provider.template_style, + '{{(' + self.when + ')|bool}}', variables = { **parameter_values, "resource_claim": resource_claim, @@ -354,10 +354,6 @@ def status_summary_template(self) -> Mapping|None: def template_enable(self): return self.spec.get('template', {}).get('enable', False) - @property - def template_style(self) -> str: - return self.spec.get('template', {}).get('style', 'jinja2') - @property def vars(self) -> Mapping: return self.spec.get('vars', {}) @@ -402,7 +398,6 @@ def apply_template_defaults(self, resource_claim, resource_index) -> Mapping: template, recursive_process_template_strings( template = self.spec['default'], - template_style = self.template_style, variables = { "resource_claim": resource_claim, "resource_index": resource_index, @@ -699,7 +694,7 @@ async def resource_definition_from_template(self, deep_merge( resource_definition, recursive_process_template_strings( - self.override, self.template_style, + self.override, variables = all_vars, template_variables = {**self.vars, **resource_handle_vars} ) diff --git a/test/unittest-poolboy_templating.py b/test/unittest-poolboy_templating.py index 2b9b83e..cac88ac 100755 --- a/test/unittest-poolboy_templating.py +++ b/test/unittest-poolboy_templating.py @@ -11,7 +11,7 @@ def test_00(self): template = {} variables = {} self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), {} ) @@ -26,7 +26,7 @@ def test_01(self): } variables = {} self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), template ) @@ -38,7 +38,7 @@ def test_02(self): "foo": "bar" } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), { "a": "bar" } @@ -52,7 +52,7 @@ def test_03(self): "foo": "bar" } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), { "a": ["bar"] } @@ -66,7 +66,7 @@ def test_04(self): "foo": True } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), { "a": "True" } @@ -80,7 +80,7 @@ def test_05(self): "foo": True } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), { "a": True } @@ -95,7 +95,7 @@ def test_06(self): "denominator": 2, } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), { "a": "0.5" } @@ -110,7 +110,7 @@ def test_07(self): "denominator": 2, } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), { "a": 0.5 } @@ -124,7 +124,7 @@ def test_08(self): "n": 21, } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), { "a": "21" } @@ -138,7 +138,7 @@ def test_09(self): "n": 21 } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), { "a": 21 } @@ -154,7 +154,7 @@ def test_10(self): } } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), { "user": "{'name': 'alice'}" } @@ -170,7 +170,7 @@ def test_11(self): } } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), + recursive_process_template_strings(template, variables), { "user": { "name": "alice" @@ -183,7 +183,7 @@ def test_12(self): "now": "{{ now() }}" } variables = {} - template_out = recursive_process_template_strings(template, 'jinja2', variables) + template_out = recursive_process_template_strings(template, variables) self.assertRegex(template_out['now'], r'^\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d+$') def test_13(self): @@ -191,7 +191,7 @@ def test_13(self): "now": "{{ now(True, '%FT%TZ') }}" } variables = {} - template_out = recursive_process_template_strings(template, 'jinja2', variables) + template_out = recursive_process_template_strings(template, variables) self.assertRegex(template_out['now'], r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ$') def test_14(self): @@ -199,7 +199,7 @@ def test_14(self): "ts": "{{ (now() + timedelta(hours=3)).strftime('%FT%TZ') }}" } variables = {} - template_out = recursive_process_template_strings(template, 'jinja2', variables) + template_out = recursive_process_template_strings(template, variables) self.assertRegex(template_out['ts'], r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ$') def test_15(self): @@ -207,7 +207,7 @@ def test_15(self): "ts": "{{ (now() + timedelta(hours=3)).strftime('%FT%TZ') }}" } variables = {} - template_out = recursive_process_template_strings(template, 'jinja2', variables) + template_out = recursive_process_template_strings(template, variables) self.assertRegex(template_out['ts'], r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ$') def test_16(self): @@ -215,7 +215,7 @@ def test_16(self): "ts": "{{ (datetime.now(timezone.utc) + timedelta(hours=3)).strftime('%FT%TZ') }}" } variables = {} - template_out = recursive_process_template_strings(template, 'jinja2', variables) + template_out = recursive_process_template_strings(template, variables) self.assertRegex(template_out['ts'], r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ$') def test_17(self): @@ -223,7 +223,7 @@ def test_17(self): "ts": "{{ (datetime.now(timezone.utc) + '3h' | parse_time_interval).strftime('%FT%TZ') }}" } variables = {} - template_out = recursive_process_template_strings(template, 'jinja2', variables) + template_out = recursive_process_template_strings(template, variables) self.assertRegex(template_out['ts'], r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ$') def test_18(self): @@ -232,7 +232,7 @@ def test_18(self): "ts2": "{{ (datetime.now(timezone.utc) + timedelta(hours=3)).strftime('%FT%TZ') }}" } variables = {} - template_out = recursive_process_template_strings(template, 'jinja2', variables) + template_out = recursive_process_template_strings(template, variables) self.assertRegex(template_out['ts1'], r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ$') self.assertEqual(template_out['ts1'], template_out['ts2']) @@ -241,7 +241,7 @@ def test_19(self): "ts": "{{ timestamp('1970-01-01T01:02:03Z').add('3h') }}", } variables = {} - template_out = recursive_process_template_strings(template, 'jinja2', variables) + template_out = recursive_process_template_strings(template, variables) self.assertEqual(template_out['ts'], '1970-01-01T04:02:03Z') def test_20(self): @@ -252,7 +252,7 @@ def test_20(self): } } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), "alice" + recursive_process_template_strings(template, variables), "alice" ) def test_21(self): @@ -269,7 +269,7 @@ def test_21(self): ] } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), ["alice", "bob"] + recursive_process_template_strings(template, variables), ["alice", "bob"] ) # Test complicated case used to determine desired state in babylon governor @@ -334,23 +334,23 @@ def test_22(self): } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), "stopped" + recursive_process_template_strings(template, variables), "stopped" ) variables['resource_templates'][0]['spec']['vars']['action_schedule']['stop'] = '2099-12-31T23:59:59Z' self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), "started" + recursive_process_template_strings(template, variables), "started" ) variables['resource_templates'][0]['spec']['vars']['action_schedule']['stop'] = '2022-01-01T00:00:00Z' del variables['resource_states'][1]['status']['towerJobs']['provision']['completeTimestamp'] self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), "started" + recursive_process_template_strings(template, variables), "started" ) del variables['resource_states'][1]['status']['towerJobs']['provision'] self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), "started" + recursive_process_template_strings(template, variables), "started" ) def test_23(self): @@ -366,7 +366,7 @@ def test_24(self): } variables = {} self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), {"a": "A"} + recursive_process_template_strings(template, variables), {"a": "A"} ) def test_25(self): @@ -375,7 +375,7 @@ def test_25(self): ] variables = {} self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), ["a", "b"] + recursive_process_template_strings(template, variables), ["a", "b"] ) def test_26(self): @@ -387,7 +387,7 @@ def test_26(self): "a": "A", } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), {"a": "A"} + recursive_process_template_strings(template, variables), {"a": "A"} ) def test_27(self): @@ -396,7 +396,7 @@ def test_27(self): "l": [{"a": "A", "b": "X"}, {"b": "B", "c": "C"}, {"d": "D"}] } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), {"a": "A", "b": "B", "c": "C", "d": "D"} + recursive_process_template_strings(template, variables), {"a": "A", "b": "B", "c": "C", "d": "D"} ) def test_28(self): @@ -405,7 +405,7 @@ def test_28(self): "l": [{"a": "A", "b": "X"}, {"b": "B", "c": "C"}, {"d": "D"}] } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), {"a": "A", "b": "B", "c": "C", "d": "D"} + recursive_process_template_strings(template, variables), {"a": "A", "b": "B", "c": "C", "d": "D"} ) def test_29(self): @@ -414,7 +414,7 @@ def test_29(self): "l": [{"a": "A", "b": "X"}, None, {"b": "B", "c": "C"}, {}, {"d": "D"}] } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), {"a": "A", "b": "B", "c": "C", "d": "D"} + recursive_process_template_strings(template, variables), {"a": "A", "b": "B", "c": "C", "d": "D"} ) def test_30(self): @@ -423,7 +423,7 @@ def test_30(self): "l": [] } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), {} + recursive_process_template_strings(template, variables), {} ) def test_31(self): @@ -432,7 +432,7 @@ def test_31(self): "l": [{"a": "A"}] } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), {"a": "A"} + recursive_process_template_strings(template, variables), {"a": "A"} ) def test_31(self): @@ -441,7 +441,7 @@ def test_31(self): "l": [{"a": "A"}] } self.assertEqual( - recursive_process_template_strings(template, 'jinja2', variables), {"a": "A"} + recursive_process_template_strings(template, variables), {"a": "A"} ) def test_32(self): @@ -452,7 +452,7 @@ def test_32(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), "A" ) @@ -465,7 +465,7 @@ def test_33(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), 2 ) @@ -477,7 +477,7 @@ def test_34(self): } with self.assertRaises(Exception): recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), def test_35(self): @@ -488,7 +488,7 @@ def test_35(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), False ) @@ -501,7 +501,7 @@ def test_36(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), 23 ) @@ -514,7 +514,7 @@ def test_37(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), 23 ) @@ -527,7 +527,7 @@ def test_38(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), True ) @@ -540,7 +540,7 @@ def test_39(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), True ) @@ -553,7 +553,7 @@ def test_40(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), True ) @@ -566,7 +566,7 @@ def test_41(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), True ) @@ -579,7 +579,7 @@ def test_42(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), True ) @@ -592,7 +592,7 @@ def test_43(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), True ) @@ -605,7 +605,7 @@ def test_44(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), True ) @@ -618,7 +618,7 @@ def test_45(self): } self.assertEqual( recursive_process_template_strings( - template, 'jinja2', template_variables=template_variables + template, template_variables=template_variables ), True )