From ef14114671f15cb975550a2b53f69b83251dd0ee Mon Sep 17 00:00:00 2001 From: Will Hegedus Date: Tue, 17 May 2022 17:00:42 -0400 Subject: [PATCH 1/4] fix: shebang on test inventory scripts /usr/bin/python is not guaranteed to exist. Should use /usr/bin/env to handle virtualenvironments and python3-only installs. --- test/f_inventory/dyninv.py | 2 +- test/f_inventory/mixeddir/dyninv.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/f_inventory/dyninv.py b/test/f_inventory/dyninv.py index 040435e..3b08cb2 100755 --- a/test/f_inventory/dyninv.py +++ b/test/f_inventory/dyninv.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python import sys diff --git a/test/f_inventory/mixeddir/dyninv.py b/test/f_inventory/mixeddir/dyninv.py index 212020a..a22f340 100755 --- a/test/f_inventory/mixeddir/dyninv.py +++ b/test/f_inventory/mixeddir/dyninv.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python import sys From e8bd4d0ac884abd72c7741300392e206355bbbc9 Mon Sep 17 00:00:00 2001 From: Will Hegedus Date: Tue, 17 May 2022 17:09:27 -0400 Subject: [PATCH 2/4] feat: Lazy loading, templated hostvars This takes advantage of lazy loading host variables from inventory so that only ones that need to be loaded by the template are actually loaded AND they go through the Jinja templater when they are loaded. --- src/ansiblecmdb/ansible_via_api.py | 78 +++++++++--------------------- 1 file changed, 23 insertions(+), 55 deletions(-) diff --git a/src/ansiblecmdb/ansible_via_api.py b/src/ansiblecmdb/ansible_via_api.py index cab3d2e..a63974e 100644 --- a/src/ansiblecmdb/ansible_via_api.py +++ b/src/ansiblecmdb/ansible_via_api.py @@ -1,6 +1,7 @@ from ansible.parsing.dataloader import DataLoader from ansible.inventory.manager import InventoryManager from ansible.vars.manager import VariableManager +from ansible.vars.hostvars import HostVars from ansiblecmdb import Ansible @@ -20,14 +21,7 @@ def load_inventories(self): loader = DataLoader() inventory = InventoryManager(loader=loader, sources=self.inventory_paths) variable_manager = VariableManager(loader=loader, inventory=inventory) - - # some Ansible variables we don't need. - ignore = ['ansible_playbook_python', - 'groups', - 'inventory_dir', - 'inventory_file', - 'omit', - 'playbook_dir'] + hostvars = HostVars(inventory, variable_manager, loader) # Handle limits here because Ansible understands more complex # limit syntax than ansible-cmdb (e.g. globbing matches []?* @@ -44,15 +38,29 @@ def load_inventories(self): del self.hosts[h] for host in inventory.get_hosts(): - vars = variable_manager.get_vars(host=host) - for key in ignore: - vars.pop(key, None) + + vars = hostvars[host.name] hostname = vars['inventory_hostname'] - groupnames = vars.pop('group_names', []) - merge_host_key_val(self.hosts, hostname, 'name', hostname) - merge_host_key_val(self.hosts, hostname, 'groups', set(groupnames)) - merge_host_key_val(self.hosts, hostname, 'hostvars', vars) + self.update_hostvars(hostname, { + 'name': hostname, + 'groups': vars['group_names'], + 'hostvars': vars + }) + + def update_hostvars(self, hostname, key_values): + """ + Update just the hostvars for a host, creating it if it did not exist + from the fact collection stage. + """ + default_empty_host = { + 'name': hostname, + 'groups': [], + 'hostvars': {} + } + host_info = self.hosts.get(hostname, default_empty_host) + host_info.update(key_values) + self.hosts[hostname] = host_info def get_hosts(self): """ @@ -61,43 +69,3 @@ def get_hosts(self): # We override this method since we already applied the limit # when we loaded the inventory. return self.hosts - - -def merge_host_key_val(hosts_dict, hostname, key, val): - """ - Update hosts_dict[`hostname`][`key`] with `val`, taking into - account all the possibilities of missing keys and merging - `val` into an existing list, set or dictionary target value. - When merging into a dict target value any matching keys will - be overwritten by the new value. Merging into a list or set - target value does not remove existing entries but instead adds - the new values to the collection. If the target value is - is not a dict or collection it will be overwritten. - - This will be called with key in ['hostvars', 'groups', 'name'], - although the implementation would work with any hashable key. - """ - if hostname not in hosts_dict: - hosts_dict[hostname] = { - 'name': hostname, - 'hostvars': {}, - 'groups': set() - } - - hostdata = hosts_dict[hostname] - if key not in hostdata: - hostdata[key] = val - return - - # We handle the list case because the analogous util.deepupdate - # does. It might be needed in deepupdate for facts, but the - # host inventory that we build is all dicts and sets. - target = hostdata[key] - if hasattr(target, 'update'): - target.update(val) # merge into target dict - elif hasattr(target, 'union'): - target.union(val) # union into target set - elif hasattr(target, 'extend'): - target.extend(val) # extend target list - else: - hostdata[key] = val # overwrite non-mergeable target value From cf5a3f90d88baa8015b6d99477750d3ee443ea10 Mon Sep 17 00:00:00 2001 From: Will Hegedus Date: Mon, 6 Jun 2022 13:24:47 -0400 Subject: [PATCH 3/4] feat: ignore inventory cache files Don't try to parse inventory cache files as if they were fact cache files. --- src/ansiblecmdb/ansible_cmdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ansiblecmdb/ansible_cmdb.py b/src/ansiblecmdb/ansible_cmdb.py index 64dc000..e575428 100644 --- a/src/ansiblecmdb/ansible_cmdb.py +++ b/src/ansiblecmdb/ansible_cmdb.py @@ -285,7 +285,8 @@ def _parse_fact_dir(self, fact_dir, fact_cache=False): break for fname in flist: - if fname.startswith('.'): + # Skip hidden files and inventory cache(s) + if fname.startswith('.') or fname.startswith('ansible_inventory_'): continue self.log.debug("Reading host facts from {0}".format(os.path.join(fact_dir, fname))) hostname = fname From 7b6ab6f66d1526b8914d179e3ca93e50483794bf Mon Sep 17 00:00:00 2001 From: Will Hegedus Date: Tue, 28 Jun 2022 14:15:35 -0400 Subject: [PATCH 4/4] feat: exclude hosts not in inventory If an inventory is specified, we should exclude discovered facts for hosts not in the inventory. This helps with situations in which old facts are cached but the host(s) is no longer in the inventory. --- src/ansiblecmdb/ansible_via_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ansiblecmdb/ansible_via_api.py b/src/ansiblecmdb/ansible_via_api.py index a63974e..e9c6f3b 100644 --- a/src/ansiblecmdb/ansible_via_api.py +++ b/src/ansiblecmdb/ansible_via_api.py @@ -32,10 +32,10 @@ def load_inventories(self): # we do the simplest thing that can work. if self.limit: inventory.subset(self.limit) - limited_hosts = inventory.get_hosts() - for h in self.hosts.keys(): - if h not in limited_hosts: - del self.hosts[h] + + # Discard facts from hosts not in the inventory from the result set + for host in set(self.hosts.keys()).difference(set(inventory.hosts.keys())): + self.hosts.pop(host) for host in inventory.get_hosts():