diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 05ee0ef..5bb3382 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -68,13 +68,13 @@ Basic Usage Before using Spyglass you must: -1. Clone the Tugboat repository: +1. Clone the Spyglass repository: .. code-block:: console git clone https://github.com/att-comdev/tugboat/tree/spyglass -2. Install the required packages in spyglass/: +2. Install the required packages in spyglass: .. code-block:: console @@ -87,24 +87,46 @@ CLI Options Usage: spyglass [OPTIONS] Options: - -s, --site TEXT Specify the site for which manifests to be - generated - -t, --type TEXT Specify the plugin type formation or tugboat - -f, --formation_url TEXT Specify the formation url - -u, --formation_user TEXT Specify the formation user id - -p, --formation_password TEXT Specify the formation user password - -d, --additional_config PATH Site specific configuraton details - -g, --generate_intermediary Dump intermediary file from passed excel and - excel spec - -m, --generate_manifests Generate manifests from the generated - intermediary file - -l, --loglevel INTEGER Loglevel NOTSET:0 ,DEBUG:10, INFO:20, - WARNING:30, ERROR:40, CRITICAL:50 [default: - 20] - --help Show this message and exit. - - -1. Running Spyglass with Remote Data Source + -s, --site TEXT Specify the site for which manifests to be + generated + -t, --type TEXT Specify the plugin type formation or tugboat + -f, --formation_url TEXT Specify the formation url + -u, --formation_user TEXT Specify the formation user id + -p, --formation_password TEXT Specify the formation user password + -i, --intermediary PATH Intermediary file path generate manifests, + use -m also with this option + -d, --additional_config PATH Site specific configuraton details + -g, --generate_intermediary Dump intermediary file from passed excel and + excel spec + -idir, --intermediary_dir PATH The path where intermediary file needs to be + generated + -e, --edit_intermediary / -nedit, --no_edit_intermediary + Flag to let user edit intermediary + -m, --generate_manifests Generate manifests from the generated + intermediary file + -mdir, --manifest_dir PATH The path where manifest files needs to be + generated + -x, --excel PATH Path to engineering excel file, to be passed + with generate_intermediary + -e, --excel_spec PATH Path to excel spec, to be passed with + generate_intermediary + -l, --loglevel INTEGER Loglevel NOTSET:0 ,DEBUG:10, INFO:20, + WARNING:30, ERROR:40, CRITICAL:50 [default: + 20] + --help Show this message and exit. + + +1. Running Spyglass with Remote Data Source Plugin spyglass -mg --type formation -f -u -p -d -s +2. Running Spyglass with Excel Plugin + +spyglass -mg --type tugboat -x -e -d -s + +for example: +spyglass -mg -t tugboat -x SiteDesignSpec_v0.1.xlsx -e excel_spec_upstream.yaml -d site_config.yaml -s airship-seaworthy +Where 'excel_spec_upstream.yaml', 'SiteDesignSpec_v0.1.xlsx' and +'site_config.yaml' are sample excel specificaton and file from +spyglass/sample folder. + diff --git a/setup.py b/setup.py index e28ec9f..adf8c6c 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ ], 'data_extractor_plugins': ['formation=spyglass.data_extractor.formation:FormationPlugin', - 'tugboat=spyglass.data_extractor.tugboat:TugboatPlugin', + 'tugboat=spyglass.data_extractor.tugboat.tugboat:TugboatPlugin', ] }, include_package_data=True, diff --git a/spyglass/config/rules.yaml b/spyglass/config/rules.yaml index 726464d..dfe4025 100644 --- a/spyglass/config/rules.yaml +++ b/spyglass/config/rules.yaml @@ -27,64 +27,12 @@ rule_ip_alloc_offset: ingress_vip: 1 static_ip_end: -2 dhcp_ip_end: -2 -rule_host_profile_interfaces: - name: host_profile_interfaces - host_profile_interfaces: - gv: - networks: - p1p1: sriov_nic01 - bond1: - - gp_nic01 - - gp_nic02 - p3p2: sriov_nic02 - nsb: - networks: - p1p1: sriov_nic01 - bond1: - - gp_nic01 - - gp_nic02 - pxe: en04 - p3p2: sriov_nic02 - cp: - networks: - p1p1: sriov_nic01 - bond1: - - gp_nic01 - - gp_nic02 - p3p2: sriov_nic02 - ns: - networks: - p1p1: sriov_nic01 - bond1: - - gp_nic01 - - gp_nic02 - pxe: en04 - p3p2: sriov_nic02 rule_hardware_profile: name: hardware_profile hardware_profile: - nc: + foundry: profile_name: - compute: nsb - ctrl: cp - host_type: - compute: nc-ns-r740 - ctrl: nc-cp - hw_type: dell_r740_purley_nc - 5ec: - profile_name: - compute: nsb - ctrl: cp - host_type: - compute: 5ec-ns-r640 - ctrl: 5ec-cp - hw_type: dell_r640_purley_5ec - cruiser: - profile_name: - compute: nsb - ctrl: cp - host_type: - compute: nc-ns-r740 - ctrl: nc-cp - hw_type: dell_r740_purley_nc + compute: dp-r720 + ctrl: cp-r720 + hw_type: dell_r720 ... diff --git a/spyglass/data_extractor/base.py b/spyglass/data_extractor/base.py index e48bd3f..a7a53da 100644 --- a/spyglass/data_extractor/base.py +++ b/spyglass/data_extractor/base.py @@ -277,7 +277,6 @@ def extract_baremetal_information(self): """ LOG.info("Extract baremetal information from plugin") baremetal = {} - is_genesis = False hosts = self.get_hosts(self.region) # For each host list fill host profile and network IPs @@ -301,40 +300,23 @@ def extract_baremetal_information(self): # Fill network IP for this host temp_host['ip'] = {} - temp_host['ip']['oob'] = temp_host_ips[host_name].get('oob', "") + temp_host['ip']['oob'] = temp_host_ips[host_name].get( + 'oob', "#CHANGE_ME") temp_host['ip']['calico'] = temp_host_ips[host_name].get( - 'calico', "") - temp_host['ip']['oam'] = temp_host_ips[host_name].get('oam', "") + 'calico', "#CHANGE_ME") + temp_host['ip']['oam'] = temp_host_ips[host_name].get( + 'oam', "#CHANGE_ME") temp_host['ip']['storage'] = temp_host_ips[host_name].get( - 'storage', "") + 'storage', "#CHANGE_ME") temp_host['ip']['overlay'] = temp_host_ips[host_name].get( - 'overlay', "") - # TODO(pg710r): Testing only. + 'overlay', "#CHANGE_ME") temp_host['ip']['pxe'] = temp_host_ips[host_name].get( 'pxe', "#CHANGE_ME") - - # TODO(nh863p): Can this logic goes into dervied plugin class - # How to determine genesis node?? - - # TODO(nh863p): If below logic is based on host profile name, then - # it should be part of design rule??? - # Filling rack_type( compute/controller/genesis) - # "cp" host profile is controller - # "ns" host profile is compute - if (temp_host['host_profile'] == 'cp'): - # The controller node is designates as genesis" - if is_genesis is False: - is_genesis = True - temp_host['type'] = 'genesis' - else: - temp_host['type'] = 'controller' - else: - temp_host['type'] = 'compute' + temp_host['type'] = host.get('type', "#CHANGE_ME") baremetal[rack_name][host_name] = temp_host LOG.debug("Baremetal information:\n{}".format( pprint.pformat(baremetal))) - return baremetal def extract_site_information(self): @@ -414,12 +396,12 @@ def extract_network_information(self): 'calico', 'overlay', 'pxe', 'storage', 'oam', 'oob', 'ingress' ] network_data['vlan_network_data'] = {} - for net in networks: tmp_net = {} if net['name'] in networks_to_scan: - tmp_net['subnet'] = net['subnet'] - tmp_net['vlan'] = net['vlan'] + tmp_net['subnet'] = net.get('subnet', '#CHANGE_ME') + if ((net['name'] != 'ingress') and (net['name'] != 'oob')): + tmp_net['vlan'] = net.get('vlan', '#CHANGE_ME') network_data['vlan_network_data'][net['name']] = tmp_net @@ -436,9 +418,10 @@ def extract_data(self): LOG.info("Extract data from plugin") site_data = {} site_data['baremetal'] = self.extract_baremetal_information() - site_data['site_info'] = self.extract_site_information() site_data['network'] = self.extract_network_information() + site_data['site_info'] = self.extract_site_information() self.site_data = site_data + return site_data def apply_additional_data(self, extra_data): @@ -450,7 +433,7 @@ def apply_additional_data(self, extra_data): If there is repetition of data then additional data supplied shall take precedence. """ - LOG.info("Update site data with additional input") + LOG.info("Merging site data with additional configuration") tmp_site_data = utils.dict_merge(self.site_data, extra_data) self.site_data = tmp_site_data return self.site_data diff --git a/spyglass/data_extractor/formation.py b/spyglass/data_extractor/formation.py index 76e9659..8a20ab9 100644 --- a/spyglass/data_extractor/formation.py +++ b/spyglass/data_extractor/formation.py @@ -63,6 +63,8 @@ def __init__(self, region): self.device_name_id_mapping = {} LOG.info("Initiated data extractor plugin:{}".format(self.source_name)) + # Implement Abstract functions + def set_config_opts(self, conf): """ Sets the config params passed by CLI""" LOG.info("Plugin params passed:\n{}".format(pprint.pformat(conf))) @@ -76,7 +78,7 @@ def set_config_opts(self, conf): self._update_site_and_zone(self.region) def get_plugin_conf(self, kwargs): - """ Validates the plugin param and return if success""" + """ Validates the plugin param from CLI and return if correct""" try: assert (kwargs['formation_url'] ) is not None, "formation_url is Not Specified" @@ -94,10 +96,210 @@ def get_plugin_conf(self, kwargs): plugin_conf = {'url': url, 'user': user, 'password': password} return plugin_conf + def get_zones(self, site=None): + zone_api = swagger_client.ZonesApi(self.formation_api_client) + + if site is None: + zones = zone_api.zones_get() + else: + site_id = self._get_site_id_by_name(site) + zones = zone_api.sites_site_id_zones_get(site_id) + + zones_list = [] + for zone in zones: + zone_name = zone.name + self.zone_name_id_mapping[zone_name] = zone.id + zones_list.append(zone_name) + + return zones_list + + def get_regions(self, zone): + zone_id = self._get_zone_id_by_name(zone) + region_api = swagger_client.RegionApi(self.formation_api_client) + regions = region_api.zones_zone_id_regions_get(zone_id) + regions_list = [] + for region in regions: + region_name = region.name + self.region_name_id_mapping[region_name] = region.id + regions_list.append(region_name) + + return regions_list + + def get_racks(self, region): + zone = self.region_zone_map[region]['zone'] + return self._get_racks(zone, rack_type='compute') + + def get_hosts(self, region, rack=None): + zone = self.region_zone_map[region]['zone'] + zone_id = self._get_zone_id_by_name(zone) + device_api = swagger_client.DevicesApi(self.formation_api_client) + control_hosts = device_api.zones_zone_id_control_nodes_get(zone_id) + compute_hosts = device_api.zones_zone_id_devices_get( + zone_id, type='KVM') + hosts_list = [] + genesis_set = False + for host in control_hosts: + self.device_name_id_mapping[host.aic_standard_name] = host.id + # The first control node is designated as genesis node + if genesis_set is False: + node_type = 'genesis' + genesis_set = True + else: + node_type = 'controller' + hosts_list.append({ + 'name': host.aic_standard_name, + 'type': node_type, + 'rack_name': host.rack_name, + 'host_profile': host.host_profile_name + }) + + for host in compute_hosts: + self.device_name_id_mapping[host.aic_standard_name] = host.id + hosts_list.append({ + 'name': host.aic_standard_name, + 'type': 'compute', + 'rack_name': host.rack_name, + 'host_profile': host.host_profile_name + }) + return hosts_list + + def get_networks(self, region): + zone = self.region_zone_map[region]['zone'] + zone_id = self._get_zone_id_by_name(zone) + region_id = self._get_region_id_by_name(region) + vlan_api = swagger_client.VlansApi(self.formation_api_client) + vlans = vlan_api.zones_zone_id_regions_region_id_vlans_get( + zone_id, region_id) + # TWEAK(pg710r):Case when vlans list is empty from + # zones_zone_id_regions_region_id_vlans_get. Ideally this should not + # be the case + if len(vlans) is 0: + # get device-id from the first host and get the network details + hosts = self.get_hosts(self.region) + host = hosts[0]['name'] + device_id = self._get_device_id_by_name(host) + vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get( + zone_id, device_id) + + LOG.debug("Extracted region network information\n{}".format(vlans)) + vlans_list = [] + for vlan_ in vlans: + if len(vlan_.vlan.ipv4) is not 0: + tmp_vlan = {} + tmp_vlan['name'] = self._get_network_name_from_vlan_name( + vlan_.vlan.name) + tmp_vlan['vlan'] = vlan_.vlan.vlan_id + tmp_vlan['subnet'] = vlan_.vlan.subnet_range + tmp_vlan['gateway'] = vlan_.ipv4_gateway + tmp_vlan['subnet_level'] = vlan_.vlan.subnet_level + vlans_list.append(tmp_vlan) + return vlans_list + + def get_ips(self, region, host=None): + zone = self.region_zone_map[region]['zone'] + zone_id = self._get_zone_id_by_name(zone) + + if host: + hosts = [host] + else: + hosts = [] + hosts_dict = self.get_hosts(zone) + for host in hosts_dict: + hosts.append(host['name']) + + vlan_api = swagger_client.VlansApi(self.formation_api_client) + ip_ = {} + + for host in hosts: + device_id = self._get_device_id_by_name(host) + vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get( + zone_id, device_id) + LOG.debug("Received VLAN Network Information\n{}".format(vlans)) + ip_[host] = {} + for vlan_ in vlans: + # The plugin currently supports IPv4 + if len(vlan_.vlan.ipv4) is not 0: + name = self._get_network_name_from_vlan_name( + vlan_.vlan.name) + ipv4 = vlan_.vlan.ipv4[0].ip + LOG.debug("vlan:{},name:{},ip:{},vlan_name:{}".format( + vlan_.vlan.vlan_id, name, ipv4, vlan_.vlan.name)) + # TODD(pg710r) This code needs to extended to support ipv4 + # and ipv6 + ip_[host][name] = ipv4 + + return ip_ + + def get_dns_servers(self, region): + try: + zone = self.region_zone_map[region]['zone'] + zone_id = self._get_zone_id_by_name(zone) + zone_api = swagger_client.ZonesApi(self.formation_api_client) + zone_ = zone_api.zones_zone_id_get(zone_id) + except swagger_client.rest.ApiException as e: + raise ApiClientError(e.msg) + + if not zone_.ipv4_dns: + LOG.warn("No dns server") + return [] + + dns_list = [] + for dns in zone_.ipv4_dns: + dns_list.append(dns.ip) + + return dns_list + + def get_ntp_servers(self, region): + # These information are not available with the formation endpoint + # These will be supplied as site config parameters + return [] + + def get_ldap_information(self, region): + # These information are not available with the formation endpoint + # These will be supplied as site config parameters + return {} + + def get_location_information(self, region): + """ get location information for a zone and return """ + site = self.region_zone_map[region]['site'] + site_id = self._get_site_id_by_name(site) + site_api = swagger_client.SitesApi(self.formation_api_client) + site_info = site_api.sites_site_id_get(site_id) + + try: + return { + # 'corridor': site_info.corridor, + 'name': site_info.city, + 'state': site_info.state, + 'country': site_info.country, + 'physical_location_id': site_info.clli, + } + except AttributeError as e: + raise MissingAttributeError('Missing {} information in {}'.format( + e, site_info.city)) + + def get_domain_name(self, region): + try: + zone = self.region_zone_map[region]['zone'] + zone_id = self._get_zone_id_by_name(zone) + zone_api = swagger_client.ZonesApi(self.formation_api_client) + zone_ = zone_api.zones_zone_id_get(zone_id) + except swagger_client.rest.ApiException as e: + raise ApiClientError(e.msg) + + if not zone_.dns: + LOG.warn('Got None while running get domain name') + return None + + return zone_.dns + + # Implement helper classes + # Functions that will be used internally within this plugin + def _validate_config_options(self, conf): - """Validate the CLI params passed + """Validate Spyglass CLI params that are related to this plugin - The method checks for missing parameters and terminates + The method checks for missing parameters for this plugin and terminates Spyglass execution if found so. """ @@ -109,10 +311,10 @@ def _validate_config_options(self, conf): LOG.error("Missing Plugin Params{}:".format(missing_params)) exit() - # Implement helper classes - def _generate_token(self): - """Generate token for Formation + """Generate token for a session with Formation endpoint + + Formation API does not provide separate resource to generate token. This is a workaround to call directly Formation API to get token instead of using Formation client. @@ -150,6 +352,7 @@ def _generate_token(self): def _get_formation_client(self): """Create formation client object + Formation uses X-Auth-Token for authentication and should be in format "user|token". Generate the token and add it formation config object. @@ -162,20 +365,15 @@ def _get_formation_client(self): def _update_site_and_zone(self, region): """Get Zone name and Site name from region""" - # TODO(nh863p): Since the test environments taking lot of time - # to retrieve data, this is a tweak to determine zone name and - # rack name - zone = region[:-1] - # TODO(pg710r): site name is hardcoded - site = zone[:-1] - - # zone = self._get_zone_by_region_name(region) - # site = self._get_site_by_zone_name(zone) + try: + zone = self._get_zone_by_region_name(region) + assert(zone is not None), "zone can't be None" + except AssertionError as e: + LOG.error("zone:None:{}".format(e)) - # TODO(nh863p): Raise exception if zone is None??? + site = self._get_site_by_zone_name(zone) self.region_zone_map[region] = {} - self.region_zone_map[region]['zone'] = zone self.region_zone_map[region]['site'] = site def _get_zone_by_region_name(self, region_name): @@ -268,159 +466,6 @@ def _get_racks(self, zone, rack_type='compute'): return racks_list - # Functions that will be used internally within this plugin - - def get_zones(self, site=None): - zone_api = swagger_client.ZonesApi(self.formation_api_client) - - if site is None: - zones = zone_api.zones_get() - else: - site_id = self._get_site_id_by_name(site) - zones = zone_api.sites_site_id_zones_get(site_id) - - zones_list = [] - for zone in zones: - zone_name = zone.name - self.zone_name_id_mapping[zone_name] = zone.id - zones_list.append(zone_name) - - return zones_list - - def get_regions(self, zone): - zone_id = self._get_zone_id_by_name(zone) - region_api = swagger_client.RegionApi(self.formation_api_client) - regions = region_api.zones_zone_id_regions_get(zone_id) - regions_list = [] - for region in regions: - region_name = region.name - self.region_name_id_mapping[region_name] = region.id - regions_list.append(region_name) - - return regions_list - - # Implement Abstract functions - - def get_racks(self, region): - zone = self.region_zone_map[region]['zone'] - return self._get_racks(zone, rack_type='compute') - - def get_hosts(self, region, rack=None): - # TODO(nh863p): Update the code to get rack wise hosts - zone = self.region_zone_map[region]['zone'] - zone_id = self._get_zone_id_by_name(zone) - device_api = swagger_client.DevicesApi(self.formation_api_client) - control_hosts = device_api.zones_zone_id_control_nodes_get(zone_id) - compute_hosts = device_api.zones_zone_id_devices_get( - zone_id, type='KVM') - - hosts_list = [] - for host in control_hosts: - self.device_name_id_mapping[host.aic_standard_name] = host.id - hosts_list.append({ - 'name': host.aic_standard_name, - 'type': 'controller', - 'rack_name': host.rack_name, - 'host_profile': host.host_profile_name - }) - - for host in compute_hosts: - self.device_name_id_mapping[host.aic_standard_name] = host.id - hosts_list.append({ - 'name': host.aic_standard_name, - 'type': 'compute', - 'rack_name': host.rack_name, - 'host_profile': host.host_profile_name - }) - """ - for host in itertools.chain(control_hosts, compute_hosts): - self.device_name_id_mapping[host.aic_standard_name] = host.id - hosts_list.append({ - 'name': host.aic_standard_name, - 'type': host.categories[0], - 'rack_name': host.rack_name, - 'host_profile': host.host_profile_name - }) - """ - - return hosts_list - - def get_networks(self, region): - zone = self.region_zone_map[region]['zone'] - zone_id = self._get_zone_id_by_name(zone) - region_id = self._get_region_id_by_name(region) - vlan_api = swagger_client.VlansApi(self.formation_api_client) - vlans = vlan_api.zones_zone_id_regions_region_id_vlans_get( - zone_id, region_id) - # Case when vlans list is empty from - # zones_zone_id_regions_region_id_vlans_get - if len(vlans) is 0: - # get device-id from the first host and get the network details - hosts = self.get_hosts(self.region) - host = hosts[0]['name'] - device_id = self._get_device_id_by_name(host) - vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get( - zone_id, device_id) - - LOG.debug("Extracted region network information\n{}".format(vlans)) - vlans_list = [] - for vlan_ in vlans: - if len(vlan_.vlan.ipv4) is not 0: - tmp_vlan = {} - tmp_vlan['name'] = self._get_network_name_from_vlan_name( - vlan_.vlan.name) - tmp_vlan['vlan'] = vlan_.vlan.vlan_id - tmp_vlan['subnet'] = vlan_.vlan.subnet_range - tmp_vlan['gateway'] = vlan_.ipv4_gateway - tmp_vlan['subnet_level'] = vlan_.vlan.subnet_level - vlans_list.append(tmp_vlan) - - # TODO(pg710r): hack to put dummy values for pxe - tmp_vlan = {} - tmp_vlan['name'] = 'pxe' - tmp_vlan['vlan'] = '43' - tmp_vlan['subnet'] = '172.30.4.0/25' - tmp_vlan['gateway'] = '172.30.4.1' - vlans_list.append(tmp_vlan) - return vlans_list - - def get_ips(self, region, host=None): - zone = self.region_zone_map[region]['zone'] - zone_id = self._get_zone_id_by_name(zone) - - if host: - hosts = [host] - else: - hosts = [] - hosts_dict = self.get_hosts(zone) - for host in hosts_dict: - hosts.append(host['name']) - - vlan_api = swagger_client.VlansApi(self.formation_api_client) - ip_ = {} - - for host in hosts: - device_id = self._get_device_id_by_name(host) - vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get( - zone_id, device_id) - LOG.debug("Received VLAN Network Information\n{}".format(vlans)) - ip_[host] = {} - for vlan_ in vlans: - # TODO(pg710r) We need to handle the case when incoming ipv4 - # list is empty - if len(vlan_.vlan.ipv4) is not 0: - name = self._get_network_name_from_vlan_name( - vlan_.vlan.name) - ipv4 = vlan_.vlan.ipv4[0].ip - LOG.debug("vlan:{},name:{},ip:{},vlan_name:{}".format( - vlan_.vlan.vlan_id, name, ipv4, vlan_.vlan.name)) - # TODD(pg710r) This code needs to extended to support ipv4 - # and ipv6 - # ip_[host][name] = {'ipv4': ipv4} - ip_[host][name] = ipv4 - - return ip_ - def _get_network_name_from_vlan_name(self, vlan_name): """ network names are ksn, oam, oob, overlay, storage, pxe @@ -430,7 +475,7 @@ def _get_network_name_from_vlan_name(self, vlan_name): vlan_name contains "server" the network name is "oam" vlan_name contains "ovs" the network name is "overlay" vlan_name contains "ILO" the network name is "oob" - TODO(pg710r): need to find out for pxe + vlan_name contains "pxe" the network name is "pxe" """ network_names = ['ksn', 'storage', 'server', 'ovs', 'ILO', 'pxe'] for name in network_names: @@ -452,62 +497,3 @@ def _get_network_name_from_vlan_name(self, vlan_name): return 'pxe' # if nothing matches return ("") - - def get_dns_servers(self, region): - try: - zone = self.region_zone_map[region]['zone'] - zone_id = self._get_zone_id_by_name(zone) - zone_api = swagger_client.ZonesApi(self.formation_api_client) - zone_ = zone_api.zones_zone_id_get(zone_id) - except swagger_client.rest.ApiException as e: - raise ApiClientError(e.msg) - - if not zone_.ipv4_dns: - LOG.warn("No dns server") - return [] - - dns_list = [] - for dns in zone_.ipv4_dns: - dns_list.append(dns.ip) - - return dns_list - - def get_ntp_servers(self, region): - return [] - - def get_ldap_information(self, region): - return {} - - def get_location_information(self, region): - """ get location information for a zone and return """ - site = self.region_zone_map[region]['site'] - site_id = self._get_site_id_by_name(site) - site_api = swagger_client.SitesApi(self.formation_api_client) - site_info = site_api.sites_site_id_get(site_id) - - try: - return { - # 'corridor': site_info.corridor, - 'name': site_info.city, - 'state': site_info.state, - 'country': site_info.country, - 'physical_location_id': site_info.clli, - } - except AttributeError as e: - raise MissingAttributeError('Missing {} information in {}'.format( - e, site_info.city)) - - def get_domain_name(self, region): - try: - zone = self.region_zone_map[region]['zone'] - zone_id = self._get_zone_id_by_name(zone) - zone_api = swagger_client.ZonesApi(self.formation_api_client) - zone_ = zone_api.zones_zone_id_get(zone_id) - except swagger_client.rest.ApiException as e: - raise ApiClientError(e.msg) - - if not zone_.dns: - LOG.warn('Got None while running get domain name') - return None - - return zone_.dns diff --git a/spyglass/data_extractor/tugboat.py b/spyglass/data_extractor/tugboat.py deleted file mode 100644 index 6706afb..0000000 --- a/spyglass/data_extractor/tugboat.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2018 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -# TODO(pg710r): The below lines will be uncommented when tugboat plugib -# code is added -""" -import pprint -import re -import requests -import swagger_client -import urllib3 - - -from spyglass.data_extractor.custom_exceptions import ( - ApiClientError, ConnectionError, MissingAttributeError, - TokenGenerationError) - -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -""" -from spyglass.data_extractor.base import BaseDataSourcePlugin -LOG = logging.getLogger(__name__) - - -class TugboatPlugin(BaseDataSourcePlugin): - def __init__(self, region): - LOG.error(" Tugboat currently not supported. Exiting!!") - exit() - - def set_config_opts(self, conf): - # TODO(pg710r): Code will be added later - pass - - def get_plugin_conf(self, kwargs): - # TODO(pg710r): Code will be added later - pass - - def get_zones(self, site=None): - # TODO(pg710r): Code will be added later - pass - - def get_regions(self, zone): - # TODO(pg710r): Code will be added later - pass - - def get_racks(self, region): - # TODO(pg710r): Code will be added later - pass - - def get_hosts(self, region, rack=None): - # TODO(pg710r): Code will be added later - pass - - def get_networks(self, region): - # TODO(pg710r): Code will be added later - pass - - def get_ips(self, region, host=None): - # TODO(pg710r): Code will be added later - pass - - def get_dns_servers(self, region): - # TODO(pg710r): Code will be added later - pass - - def get_ntp_servers(self, region): - # TODO(pg710r): Code will be added later - pass - - def get_ldap_information(self, region): - # TODO(pg710r): Code will be added later - pass - - def get_location_information(self, region): - # TODO(pg710r): Code will be added later - pass - - def get_domain_name(self, region): - # TODO(pg710r): Code will be added later - pass diff --git a/spyglass/data_extractor/tugboat/check_exceptions.py b/spyglass/data_extractor/tugboat/check_exceptions.py new file mode 100644 index 0000000..d11d58a --- /dev/null +++ b/spyglass/data_extractor/tugboat/check_exceptions.py @@ -0,0 +1,35 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BaseError(Exception): + pass + + +class NotEnoughIp(BaseError): + def __init__(self, cidr, total_nodes): + self.cidr = cidr + self.total_nodes = total_nodes + + def display_error(self): + print('{} can not handle {} nodes'.format(self.cidr, self.total_nodes)) + + +class NoSpecMatched(BaseError): + def __init__(self, excel_specs): + self.specs = excel_specs + + def display_error(self): + print('No spec matched. Following are the available specs:\n'.format( + self.specs)) diff --git a/spyglass/data_extractor/tugboat/excel_parser.py b/spyglass/data_extractor/tugboat/excel_parser.py new file mode 100644 index 0000000..d19ea68 --- /dev/null +++ b/spyglass/data_extractor/tugboat/excel_parser.py @@ -0,0 +1,410 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import pprint +import re +import sys +import yaml +from openpyxl import load_workbook +from openpyxl import Workbook +from .check_exceptions import ( + NoSpecMatched, ) +# from spyglass.data_extractor.custom_exceptions + +LOG = logging.getLogger(__name__) + + +class ExcelParser(): + """ Parse data from excel into a dict """ + + def __init__(self, file_name, excel_specs): + self.file_name = file_name + with open(excel_specs, 'r') as f: + spec_raw_data = f.read() + self.excel_specs = yaml.safe_load(spec_raw_data) + # A combined design spec, returns a workbok object after combining + # all the inputs excel specs + combined_design_spec = self.combine_excel_design_specs(file_name) + self.wb_combined = combined_design_spec + self.filenames = file_name + self.spec = 'xl_spec' + + @staticmethod + def sanitize(string): + """ Remove extra spaces and convert string to lower case """ + return string.replace(' ', '').lower() + + def compare(self, string1, string2): + """ Compare the strings """ + return bool(re.search(self.sanitize(string1), self.sanitize(string2))) + + def validate_sheet(self, spec, sheet): + """ Check if the sheet is correct or not """ + ws = self.wb_combined[sheet] + header_row = self.excel_specs['specs'][spec]['header_row'] + ipmi_header = self.excel_specs['specs'][spec]['ipmi_address_header'] + ipmi_column = self.excel_specs['specs'][spec]['ipmi_address_col'] + header_value = ws.cell(row=header_row, column=ipmi_column).value + return bool(self.compare(ipmi_header, header_value)) + + def find_correct_spec(self): + """ Find the correct spec """ + for spec in self.excel_specs['specs']: + sheet_name = self.excel_specs['specs'][spec]['ipmi_sheet_name'] + for sheet in self.wb_combined.sheetnames: + if self.compare(sheet_name, sheet): + self.excel_specs['specs'][spec]['ipmi_sheet_name'] = sheet + if self.validate_sheet(spec, sheet): + return spec + raise NoSpecMatched(self.excel_specs) + + def get_ipmi_data(self): + """ Read IPMI data from the sheet """ + ipmi_data = {} + hosts = [] + provided_sheetname = self.excel_specs['specs'][self. + spec]['ipmi_sheet_name'] + workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( + provided_sheetname) + if workbook_object is not None: + ws = workbook_object[extracted_sheetname] + else: + ws = self.wb_combined[provided_sheetname] + row = self.excel_specs['specs'][self.spec]['start_row'] + end_row = self.excel_specs['specs'][self.spec]['end_row'] + hostname_col = self.excel_specs['specs'][self.spec]['hostname_col'] + ipmi_address_col = self.excel_specs['specs'][self. + spec]['ipmi_address_col'] + host_profile_col = self.excel_specs['specs'][self. + spec]['host_profile_col'] + ipmi_gateway_col = self.excel_specs['specs'][self. + spec]['ipmi_gateway_col'] + previous_server_gateway = None + while row <= end_row: + hostname = self.sanitize( + ws.cell(row=row, column=hostname_col).value) + hosts.append(hostname) + ipmi_address = ws.cell(row=row, column=ipmi_address_col).value + if '/' in ipmi_address: + ipmi_address = ipmi_address.split('/')[0] + ipmi_gateway = ws.cell(row=row, column=ipmi_gateway_col).value + if ipmi_gateway: + previous_server_gateway = ipmi_gateway + else: + ipmi_gateway = previous_server_gateway + host_profile = ws.cell(row=row, column=host_profile_col).value + try: + if host_profile is None: + raise RuntimeError("No value read from {} ".format( + self.file_name) + "sheet:{} row:{}, col:{}".format( + self.spec, row, host_profile_col)) + except RuntimeError as rerror: + LOG.critical(rerror) + sys.exit("Tugboat exited!!") + ipmi_data[hostname] = { + 'ipmi_address': ipmi_address, + 'ipmi_gateway': ipmi_gateway, + 'host_profile': host_profile, + 'type': type, + } + row += 1 + LOG.debug("ipmi data extracted from excel:\n{}".format( + pprint.pformat(ipmi_data))) + LOG.debug("host data extracted from excel:\n{}".format( + pprint.pformat(hosts))) + return [ipmi_data, hosts] + + def get_private_vlan_data(self, ws): + """ Get private vlan data from private IP sheet """ + vlan_data = {} + row = self.excel_specs['specs'][self.spec]['vlan_start_row'] + end_row = self.excel_specs['specs'][self.spec]['vlan_end_row'] + type_col = self.excel_specs['specs'][self.spec]['net_type_col'] + vlan_col = self.excel_specs['specs'][self.spec]['vlan_col'] + while row <= end_row: + cell_value = ws.cell(row=row, column=type_col).value + if cell_value: + vlan = ws.cell(row=row, column=vlan_col).value + if vlan: + vlan = vlan.lower() + vlan_data[vlan] = cell_value + row += 1 + LOG.debug("vlan data extracted from excel:\n%s", + pprint.pformat(vlan_data)) + return vlan_data + + def get_private_network_data(self): + """ Read network data from the private ip sheet """ + provided_sheetname = self.excel_specs['specs'][ + self.spec]['private_ip_sheet'] + workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( + provided_sheetname) + if workbook_object is not None: + ws = workbook_object[extracted_sheetname] + else: + ws = self.wb_combined[provided_sheetname] + vlan_data = self.get_private_vlan_data(ws) + network_data = {} + row = self.excel_specs['specs'][self.spec]['net_start_row'] + end_row = self.excel_specs['specs'][self.spec]['net_end_row'] + col = self.excel_specs['specs'][self.spec]['net_col'] + vlan_col = self.excel_specs['specs'][self.spec]['net_vlan_col'] + old_vlan = '' + while row <= end_row: + vlan = ws.cell(row=row, column=vlan_col).value + if vlan: + vlan = vlan.lower() + network = ws.cell(row=row, column=col).value + if vlan and network: + net_type = vlan_data[vlan] + if 'vlan' not in network_data: + network_data[net_type] = { + 'vlan': vlan, + 'subnet': [], + } + elif not vlan and network: + # If vlan is not present then assign old vlan to vlan as vlan + # value is spread over several rows + vlan = old_vlan + else: + row += 1 + continue + network_data[vlan_data[vlan]]['subnet'].append(network) + old_vlan = vlan + row += 1 + for network in network_data: + network_data[network]['is_common'] = True + """ + if len(network_data[network]['subnet']) > 1: + network_data[network]['is_common'] = False + else: + network_data[network]['is_common'] = True + LOG.debug( + "private network data extracted from\ + excel:\n%s", pprint.pformat(network_data)) + """ + return network_data + + def get_public_network_data(self): + """ Read public network data from public ip data """ + network_data = {} + provided_sheetname = self.excel_specs['specs'][self. + spec]['public_ip_sheet'] + workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( + provided_sheetname) + if workbook_object is not None: + ws = workbook_object[extracted_sheetname] + else: + ws = self.wb_combined[provided_sheetname] + oam_row = self.excel_specs['specs'][self.spec]['oam_ip_row'] + oam_col = self.excel_specs['specs'][self.spec]['oam_ip_col'] + oam_vlan_col = self.excel_specs['specs'][self.spec]['oam_vlan_col'] + ingress_row = self.excel_specs['specs'][self.spec]['ingress_ip_row'] + oob_row = self.excel_specs['specs'][self.spec]['oob_net_row'] + col = self.excel_specs['specs'][self.spec]['oob_net_start_col'] + end_col = self.excel_specs['specs'][self.spec]['oob_net_end_col'] + network_data = { + 'oam': { + 'subnet': [ws.cell(row=oam_row, column=oam_col).value], + 'vlan': ws.cell(row=oam_row, column=oam_vlan_col).value, + }, + 'ingress': ws.cell(row=ingress_row, column=oam_col).value, + } + network_data['oob'] = { + 'subnet': [], + } + while col <= end_col: + cell_value = ws.cell(row=oob_row, column=col).value + if cell_value: + network_data['oob']['subnet'].append(self.sanitize(cell_value)) + col += 1 + LOG.debug( + "public network data extracted from\ + excel:\n%s", pprint.pformat(network_data)) + return network_data + + def get_site_info(self): + """ Read location, dns, ntp and ldap data""" + site_info = {} + provided_sheetname = self.excel_specs['specs'][ + self.spec]['dns_ntp_ldap_sheet'] + workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( + provided_sheetname) + if workbook_object is not None: + ws = workbook_object[extracted_sheetname] + else: + ws = self.wb_combined[provided_sheetname] + dns_row = self.excel_specs['specs'][self.spec]['dns_row'] + dns_col = self.excel_specs['specs'][self.spec]['dns_col'] + ntp_row = self.excel_specs['specs'][self.spec]['ntp_row'] + ntp_col = self.excel_specs['specs'][self.spec]['ntp_col'] + domain_row = self.excel_specs['specs'][self.spec]['domain_row'] + domain_col = self.excel_specs['specs'][self.spec]['domain_col'] + login_domain_row = self.excel_specs['specs'][self. + spec]['login_domain_row'] + ldap_col = self.excel_specs['specs'][self.spec]['ldap_col'] + global_group = self.excel_specs['specs'][self.spec]['global_group'] + ldap_search_url_row = self.excel_specs['specs'][ + self.spec]['ldap_search_url_row'] + dns_servers = ws.cell(row=dns_row, column=dns_col).value + ntp_servers = ws.cell(row=ntp_row, column=ntp_col).value + try: + if dns_servers is None: + raise RuntimeError( + "No value for dns_server from:{} Sheet:'{}' Row:{} Col:{}". + format(self.file_name, provided_sheetname, dns_row, + dns_col)) + raise RuntimeError( + "No value for ntp_server frome:{} Sheet:'{}' Row:{} Col:{}" + .format(self.file_name, provided_sheetname, ntp_row, + ntp_col)) + except RuntimeError as rerror: + LOG.critical(rerror) + sys.exit("Tugboat exited!!") + + dns_servers = dns_servers.replace('\n', ' ') + ntp_servers = ntp_servers.replace('\n', ' ') + if ',' in dns_servers: + dns_servers = dns_servers.split(',') + else: + dns_servers = dns_servers.split() + if ',' in ntp_servers: + ntp_servers = ntp_servers.split(',') + else: + ntp_servers = ntp_servers.split() + site_info = { + 'location': self.get_location_data(), + 'dns': dns_servers, + 'ntp': ntp_servers, + 'domain': ws.cell(row=domain_row, column=domain_col).value, + 'ldap': { + 'subdomain': ws.cell(row=login_domain_row, + column=ldap_col).value, + 'common_name': ws.cell(row=global_group, + column=ldap_col).value, + 'url': ws.cell(row=ldap_search_url_row, column=ldap_col).value, + } + } + LOG.debug( + "Site Info extracted from\ + excel:\n%s", pprint.pformat(site_info)) + return site_info + + def get_location_data(self): + """ Read location data from the site and zone sheet """ + provided_sheetname = self.excel_specs['specs'][self. + spec]['location_sheet'] + workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( + provided_sheetname) + if workbook_object is not None: + ws = workbook_object[extracted_sheetname] + else: + ws = self.wb_combined[provided_sheetname] + corridor_row = self.excel_specs['specs'][self.spec]['corridor_row'] + column = self.excel_specs['specs'][self.spec]['column'] + site_name_row = self.excel_specs['specs'][self.spec]['site_name_row'] + state_name_row = self.excel_specs['specs'][self.spec]['state_name_row'] + country_name_row = self.excel_specs['specs'][self. + spec]['country_name_row'] + clli_name_row = self.excel_specs['specs'][self.spec]['clli_name_row'] + return { + 'corridor': ws.cell(row=corridor_row, column=column).value, + 'name': ws.cell(row=site_name_row, column=column).value, + 'state': ws.cell(row=state_name_row, column=column).value, + 'country': ws.cell(row=country_name_row, column=column).value, + 'physical_location': ws.cell(row=clli_name_row, + column=column).value, + } + + def validate_sheet_names_with_spec(self): + """ Checks is sheet name in spec file matches with excel file""" + spec = list(self.excel_specs['specs'].keys())[0] + spec_item = self.excel_specs['specs'][spec] + sheet_name_list = [] + ipmi_header_sheet_name = spec_item['ipmi_sheet_name'] + sheet_name_list.append(ipmi_header_sheet_name) + private_ip_sheet_name = spec_item['private_ip_sheet'] + sheet_name_list.append(private_ip_sheet_name) + public_ip_sheet_name = spec_item['public_ip_sheet'] + sheet_name_list.append(public_ip_sheet_name) + dns_ntp_ldap_sheet_name = spec_item['dns_ntp_ldap_sheet'] + sheet_name_list.append(dns_ntp_ldap_sheet_name) + location_sheet_name = spec_item['location_sheet'] + sheet_name_list.append(location_sheet_name) + try: + for sheetname in sheet_name_list: + workbook_object, extracted_sheetname = \ + self.get_xl_obj_and_sheetname(sheetname) + if workbook_object is not None: + wb = workbook_object + sheetname = extracted_sheetname + else: + wb = self.wb_combined + + if sheetname not in wb.sheetnames: + raise RuntimeError( + "SheetName '{}' not found ".format(sheetname)) + except RuntimeError as rerror: + LOG.critical(rerror) + sys.exit("Tugboat exited!!") + + LOG.info("Sheet names in excel spec validated") + + def get_data(self): + """ Create a dict with combined data """ + self.validate_sheet_names_with_spec() + ipmi_data = self.get_ipmi_data() + network_data = self.get_private_network_data() + public_network_data = self.get_public_network_data() + site_info_data = self.get_site_info() + data = { + 'ipmi_data': ipmi_data, + 'network_data': { + 'private': network_data, + 'public': public_network_data, + }, + 'site_info': site_info_data, + } + LOG.debug( + "Location data extracted from\ + excel:\n%s", pprint.pformat(data)) + return data + + def combine_excel_design_specs(self, filenames): + """ Combines multiple excel file to a single design spec""" + design_spec = Workbook() + for exel_file in filenames: + loaded_workbook = load_workbook(exel_file, data_only=True) + for names in loaded_workbook.sheetnames: + design_spec_worksheet = design_spec.create_sheet(names) + loaded_workbook_ws = loaded_workbook[names] + for row in loaded_workbook_ws: + for cell in row: + design_spec_worksheet[cell. + coordinate].value = cell.value + return design_spec + + def get_xl_obj_and_sheetname(self, sheetname): + """ + The logic confirms if the sheetname is specified for example as: + "MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs" + """ + if (re.search('.xlsx', sheetname) or re.search('.xls', sheetname)): + """ Extract file name """ + source_xl_file = sheetname.split(':')[0] + wb = load_workbook(source_xl_file, data_only=True) + return [wb, sheetname.split(':')[1]] + else: + return [None, sheetname] diff --git a/spyglass/data_extractor/tugboat/tugboat.py b/spyglass/data_extractor/tugboat/tugboat.py new file mode 100644 index 0000000..4a92657 --- /dev/null +++ b/spyglass/data_extractor/tugboat/tugboat.py @@ -0,0 +1,350 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import logging +import pprint +import re +from spyglass.data_extractor.base import BaseDataSourcePlugin +from spyglass.data_extractor.tugboat.excel_parser import ExcelParser + +LOG = logging.getLogger(__name__) + + +class TugboatPlugin(BaseDataSourcePlugin): + def __init__(self, region): + LOG.info("Tugboat Initializing") + self.source_type = 'excel' + self.source_name = 'tugboat' + + # Configuration parameters + self.excel_path = None + self.excel_spec = None + + # Site related data + self.region = region + + # Raw data from excel + self.parsed_xl_data = None + + LOG.info("Initiated data extractor plugin:{}".format(self.source_name)) + + def set_config_opts(self, conf): + """ + Placeholder to set confgiuration options + specific to each plugin. + + :param dict conf: Configuration options as dict + + Example: conf = { 'excel_spec': 'spec1.yaml', + 'excel_path': 'excel.xls' } + + Each plugin will have their own config opts. + """ + self.excel_path = conf['excel_path'] + self.excel_spec = conf['excel_spec'] + + # Extract raw data from excel sheets + self._get_excel_obj() + self._extract_raw_data_from_excel() + return + + def get_plugin_conf(self, kwargs): + """ Validates the plugin param from CLI and return if correct + + + Ideally the CLICK module shall report an error if excel file + and excel specs are not specified. The below code has been + written as an additional safeguard. + """ + try: + assert (len( + kwargs['excel'])), "Engineering Spec file not specified" + excel_file_info = kwargs['excel'] + assert (kwargs['excel_spec'] + ) is not None, "Excel Spec file not specified" + excel_spec_info = kwargs['excel_spec'] + except AssertionError as e: + LOG.error("{}:Spyglass exited!".format(e)) + exit() + plugin_conf = { + 'excel_path': excel_file_info, + 'excel_spec': excel_spec_info + } + return plugin_conf + + def get_hosts(self, region, rack=None): + """Return list of hosts in the region + :param string region: Region name + :param string rack: Rack name + :returns: list of hosts information + :rtype: list of dict + Example: [ + { + 'name': 'host01', + 'type': 'controller', + 'host_profile': 'hp_01' + }, + { + 'name': 'host02', + 'type': 'compute', + 'host_profile': 'hp_02'} + ] + """ + LOG.info("Get Host Information") + ipmi_data = self.parsed_xl_data['ipmi_data'][0] + rackwise_hosts = self._get_rackwise_hosts() + host_list = [] + for rack in rackwise_hosts.keys(): + for host in rackwise_hosts[rack]: + host_list.append({ + 'rack_name': + rack, + 'name': + host, + 'host_profile': + ipmi_data[host]['host_profile'] + }) + return host_list + + def get_networks(self, region): + """ Extracts vlan network info from raw network data from excel""" + vlan_list = [] + # Network data extracted from xl is formatted to have a predictable + # data type. For e.g VlAN 45 extracted from xl is formatted as 45 + vlan_pattern = r'\d+' + private_net = self.parsed_xl_data['network_data']['private'] + public_net = self.parsed_xl_data['network_data']['public'] + # Extract network information from private and public network data + for net_type, net_val in itertools.chain(private_net.items(), + public_net.items()): + tmp_vlan = {} + # Ingress is special network that has no vlan, only a subnet string + # So treatment for ingress is different + if net_type is not 'ingress': + # standardize the network name as net_type may ne different. + # For e.g insteas of pxe it may be PXE or instead of calico + # it may be ksn. Valid network names are pxe, calico, oob, oam, + # overlay, storage, ingress + tmp_vlan['name'] = self._get_network_name_from_vlan_name( + net_type) + + # extract vlan tag. It was extracted from xl file as 'VlAN 45' + # The code below extracts the numeric data fron net_val['vlan'] + if net_val.get('vlan', "") is not "": + value = re.findall(vlan_pattern, net_val['vlan']) + tmp_vlan['vlan'] = value[0] + else: + tmp_vlan['vlan'] = "#CHANGE_ME" + + tmp_vlan['subnet'] = net_val.get('subnet', "#CHANGE_ME") + tmp_vlan['gateway'] = net_val.get('gateway', "#CHANGE_ME") + else: + tmp_vlan['name'] = 'ingress' + tmp_vlan['subnet'] = net_val + vlan_list.append(tmp_vlan) + LOG.debug("vlan list extracted from tugboat:\n{}".format( + pprint.pformat(vlan_list))) + return vlan_list + + def get_ips(self, region, host=None): + """Return list of IPs on the host + :param string region: Region name + :param string host: Host name + :returns: Dict of IPs per network on the host + :rtype: dict + Example: {'oob': {'ipv4': '192.168.1.10'}, + 'pxe': {'ipv4': '192.168.2.10'}} + The network name from get_networks is expected to be the keys of this + dict. In case some networks are missed, they are expected to be either + DHCP or internally generated n the next steps by the design rules. + """ + + ip_ = {} + ipmi_data = self.parsed_xl_data['ipmi_data'][0] + ip_[host] = { + 'oob': ipmi_data[host].get('ipmi_address', '#CHANGE_ME'), + 'oam': ipmi_data[host].get('oam', '#CHANGE_ME'), + 'calico': ipmi_data[host].get('calico', '#CHANGE_ME'), + 'overlay': ipmi_data[host].get('overlay', '#CHANGE_ME'), + 'pxe': ipmi_data[host].get('pxe', '#CHANGE_ME'), + 'storage': ipmi_data[host].get('storage', '#CHANGE_ME') + } + return ip_ + + def get_ldap_information(self, region): + """ Extract ldap information from excel""" + + ldap_raw_data = self.parsed_xl_data['site_info']['ldap'] + ldap_info = {} + # raw url is 'url: ldap://example.com' so we are converting to + # 'ldap://example.com' + url = ldap_raw_data.get('url', '#CHANGE_ME') + try: + ldap_info['url'] = url.split(' ')[1] + ldap_info['domain'] = url.split('.')[1] + except IndexError as e: + LOG.error("url.split:{}".format(e)) + ldap_info['common_name'] = ldap_raw_data.get('common_name', + '#CHANGE_ME') + ldap_info['subdomain'] = ldap_raw_data.get('subdomain', '#CHANGE_ME') + + return ldap_info + + def get_ntp_servers(self, region): + """ Returns a comma separated list of ntp ip addresses""" + + ntp_server_list = self._get_formatted_server_list( + self.parsed_xl_data['site_info']['ntp']) + return ntp_server_list + + def get_dns_servers(self, region): + """ Returns a comma separated list of dns ip addresses""" + dns_server_list = self._get_formatted_server_list( + self.parsed_xl_data['site_info']['dns']) + return dns_server_list + + def get_domain_name(self, region): + """ Returns domain name extracted from excel file""" + + return self.parsed_xl_data['site_info']['domain'] + + def get_location_information(self, region): + """ + Prepare location data from information extracted + by ExcelParser(i.e raw data) + """ + location_data = self.parsed_xl_data['site_info']['location'] + + corridor_pattern = r'\d+' + corridor_number = re.findall(corridor_pattern, + location_data['corridor'])[0] + name = location_data.get('name', '#CHANGE_ME') + state = location_data.get('state', '#CHANGE_ME') + country = location_data.get('country', '#CHANGE_ME') + physical_location_id = location_data.get('physical_location', '') + + return { + 'name': name, + 'physical_location_id': physical_location_id, + 'state': state, + 'country': country, + 'corridor': 'c{}'.format(corridor_number), + } + + def get_racks(self, region): + # This function is not required since the excel plugin + # already provide rack information. + pass + + def _get_excel_obj(self): + """ Creation of an ExcelParser object to store site information. + + The information is obtained based on a excel spec yaml file. + This spec contains row, column and sheet information of + the excel file from where site specific data can be extracted. + """ + self.excel_obj = ExcelParser(self.excel_path, self.excel_spec) + + def _extract_raw_data_from_excel(self): + """ Extracts raw information from excel file based on excel spec""" + self.parsed_xl_data = self.excel_obj.get_data() + + def _get_network_name_from_vlan_name(self, vlan_name): + """ network names are ksn, oam, oob, overlay, storage, pxe + + + This is a utility function to determine the vlan acceptable + vlan from the name extracted from excel file + + The following mapping rules apply: + vlan_name contains "ksn or calico" the network name is "calico" + vlan_name contains "storage" the network name is "storage" + vlan_name contains "server" the network name is "oam" + vlan_name contains "ovs" the network name is "overlay" + vlan_name contains "oob" the network name is "oob" + vlan_name contains "pxe" the network name is "pxe" + """ + network_names = [ + 'ksn|calico', 'storage', 'oam|server', 'ovs|overlay', 'oob', 'pxe' + ] + for name in network_names: + # Make a pattern that would ignore case. + # if name is 'ksn' pattern name is '(?i)(ksn)' + name_pattern = "(?i)({})".format(name) + if re.search(name_pattern, vlan_name): + if name is 'ksn|calico': + return 'calico' + if name is 'storage': + return 'storage' + if name is 'oam|server': + return 'oam' + if name is 'ovs|overlay': + return 'overlay' + if name is 'oob': + return 'oob' + if name is 'pxe': + return 'pxe' + # if nothing matches + LOG.error( + "Unable to recognize VLAN name extracted from Plugin data source") + return ("") + + def _get_formatted_server_list(self, server_list): + """ Format dns and ntp server list as comma separated string """ + + # dns/ntp server info from excel is of the format + # 'xxx.xxx.xxx.xxx, (aaa.bbb.ccc.com)' + # The function returns a list of comma separated dns ip addresses + servers = [] + for data in server_list: + if '(' not in data: + servers.append(data) + formatted_server_list = ','.join(servers) + return formatted_server_list + + def _get_rack(self, host): + """ + Get rack id from the rack string extracted + from xl + """ + rack_pattern = r'\w.*(r\d+)\w.*' + rack = re.findall(rack_pattern, host)[0] + if not self.region: + self.region = host.split(rack)[0] + return rack + + def _get_rackwise_hosts(self): + """ Mapping hosts with rack ids """ + rackwise_hosts = {} + hostnames = self.parsed_xl_data['ipmi_data'][1] + racks = self._get_rack_data() + for rack in racks: + if rack not in rackwise_hosts: + rackwise_hosts[racks[rack]] = [] + for host in hostnames: + if rack in host: + rackwise_hosts[racks[rack]].append(host) + LOG.debug("rackwise hosts:\n%s", pprint.pformat(rackwise_hosts)) + return rackwise_hosts + + def _get_rack_data(self): + """ Format rack name """ + LOG.info("Getting rack data") + racks = {} + hostnames = self.parsed_xl_data['ipmi_data'][1] + for host in hostnames: + rack = self._get_rack(host) + racks[rack] = rack.replace('r', 'rack') + return racks diff --git a/spyglass/parser/generate_intermediary.py b/spyglass/parser/generate_intermediary.py index b238333..0145d9c 100644 --- a/spyglass/parser/generate_intermediary.py +++ b/spyglass/parser/generate_intermediary.py @@ -52,44 +52,47 @@ def _initialize_intermediary(self): self.sitetype = None self.genesis_node = None self.region_name = None + self.network_subnets = None def _get_network_subnets(self): - # Extract subnet information for networks + """ Extract subnet information for networks. + + + In some networks, there are multiple subnets, in that case + we assign only the first subnet """ LOG.info("Extracting network subnets") network_subnets = {} - # self.format_network_data() for net_type in self.data['network']['vlan_network_data']: # One of the type is ingress and we don't want that here if (net_type != 'ingress'): network_subnets[net_type] = netaddr.IPNetwork( self.data['network']['vlan_network_data'][net_type] - ['subnet']) + ['subnet'][0]) LOG.debug("Network subnets:\n{}".format( pprint.pformat(network_subnets))) return network_subnets def _get_genesis_node_details(self): - # Returns the genesis node details - LOG.info("Getting Genesis Node Details") + # Get genesis host node details from the hosts based on host type for racks in self.data['baremetal'].keys(): rack_hosts = self.data['baremetal'][racks] for host in rack_hosts: if rack_hosts[host]['type'] == 'genesis': self.genesis_node = rack_hosts[host] self.genesis_node['name'] = host - LOG.debug("Genesis Node Details:{}".format( + LOG.debug("Genesis Node Details:\n{}".format( pprint.pformat(self.genesis_node))) - def _validate_extracted_data(self, data): - """ Validates the extracted data from input source. + def _validate_intermediary_data(self, data): + """ Validates the intermediary data before generating manifests. It checks wether the data types and data format are as expected. The method validates this with regex pattern defined for each data type. """ - LOG.info('Validating data read from extracted data') + LOG.info('Validating Intermediary data') temp_data = {} # Peforming a deep copy temp_data = copy.deepcopy(data) @@ -153,14 +156,75 @@ def _apply_design_rules(self): LOG.info("Applying rule:{}".format(rule_name)) def _apply_rule_host_profile_interfaces(self, rule_data): - # TODO(pg710r)Nothing to do as of now + # TODO(pg710r)Nothing to do as of now since host profile + # information is already present in plugin data. + # This function shall be defined if plugin data source + # doesn't provide host profile information. pass def _apply_rule_hardware_profile(self, rule_data): - # TODO(pg710r)Nothing to do as of now - pass + """ Apply rules to define host type from hardware profile info. + + + Host profile will define host types as "controller, compute or + genesis". The rule_data has pre-defined information to define + compute or controller based on host_profile. For defining 'genesis' + the first controller host is defined as genesis.""" + is_genesis = False + hardware_profile = rule_data[self.data['site_info']['sitetype']] + # Getting individual racks. The racks are sorted to ensure that the + # first controller of the first rack is assigned as 'genesis' node. + for rack in sorted(self.data['baremetal'].keys()): + # Getting individual hosts in each rack. Sorting of the hosts are + # done to determine the genesis node. + for host in sorted(self.data['baremetal'][rack].keys()): + host_info = self.data['baremetal'][rack][host] + if (host_info['host_profile'] == hardware_profile[ + 'profile_name']['ctrl']): + if not is_genesis: + host_info['type'] = 'genesis' + is_genesis = True + else: + host_info['type'] = 'controller' + else: + host_info['type'] = 'compute' def _apply_rule_ip_alloc_offset(self, rule_data): + """ Apply offset rules to update baremetal host ip's and vlan network + data """ + + # Get network subnets + self.network_subnets = self._get_network_subnets() + + self._update_vlan_net_data(rule_data) + self._update_baremetal_host_ip_data(rule_data) + + def _update_baremetal_host_ip_data(self, rule_data): + """ Update baremetal host ip's for applicable networks. + + + The applicable networks are oob, oam, ksn, storage and overlay. + These IPs are assigned based on network subnets ranges. + If a particular ip exists it is overridden.""" + + # Ger defult ip offset + default_ip_offset = rule_data['default'] + + host_idx = 0 + LOG.info("Update baremetal host ip's") + for racks in self.data['baremetal'].keys(): + rack_hosts = self.data['baremetal'][racks] + for host in rack_hosts: + host_networks = rack_hosts[host]['ip'] + for net in host_networks: + ips = list(self.network_subnets[net]) + host_networks[net] = str(ips[host_idx + default_ip_offset]) + host_idx = host_idx + 1 + + LOG.debug("Updated baremetal host:\n{}".format( + pprint.pformat(self.data['baremetal']))) + + def _update_vlan_net_data(self, rule_data): """ Offset allocation rules to determine ip address range(s) @@ -168,7 +232,6 @@ def _apply_rule_ip_alloc_offset(self, rule_data): network address, gateway ip and other address ranges """ LOG.info("Apply network design rules") - vlan_network_data = {} # Collect Rules default_ip_offset = rule_data['default'] @@ -181,7 +244,7 @@ def _apply_rule_ip_alloc_offset(self, rule_data): dhcp_ip_end_offset = rule_data['dhcp_ip_end'] # Set ingress vip and CIDR for bgp - LOG.info("Applying rule to network bgp data") + LOG.info("Apply network design rules:bgp") subnet = netaddr.IPNetwork( self.data['network']['vlan_network_data']['ingress']['subnet'][0]) ips = list(subnet) @@ -192,27 +255,24 @@ def _apply_rule_ip_alloc_offset(self, rule_data): LOG.debug("Updated network bgp data:\n{}".format( pprint.pformat(self.data['network']['bgp']))) - LOG.info("Applying rule to vlan network data") - # Get network subnets - network_subnets = self._get_network_subnets() + LOG.info("Apply network design rules:vlan") # Apply rules to vlan networks - for net_type in network_subnets: + for net_type in self.network_subnets: if net_type == 'oob': ip_offset = oob_ip_offset else: ip_offset = default_ip_offset - vlan_network_data[net_type] = {} - subnet = network_subnets[net_type] - ips = list(subnet) - vlan_network_data[net_type]['network'] = str( - network_subnets[net_type]) + subnet = self.network_subnets[net_type] + ips = list(subnet) - vlan_network_data[net_type]['gateway'] = str( - ips[gateway_ip_offset]) + self.data['network']['vlan_network_data'][net_type][ + 'gateway'] = str(ips[gateway_ip_offset]) - vlan_network_data[net_type]['reserved_start'] = str(ips[1]) - vlan_network_data[net_type]['reserved_end'] = str(ips[ip_offset]) + self.data['network']['vlan_network_data'][net_type][ + 'reserved_start'] = str(ips[1]) + self.data['network']['vlan_network_data'][net_type][ + 'reserved_end'] = str(ips[ip_offset]) static_start = str(ips[ip_offset + 1]) static_end = str(ips[static_ip_end_offset]) @@ -223,30 +283,32 @@ def _apply_rule_ip_alloc_offset(self, rule_data): dhcp_start = str(ips[mid]) dhcp_end = str(ips[dhcp_ip_end_offset]) - vlan_network_data[net_type]['dhcp_start'] = dhcp_start - vlan_network_data[net_type]['dhcp_end'] = dhcp_end + self.data['network']['vlan_network_data'][net_type][ + 'dhcp_start'] = dhcp_start + self.data['network']['vlan_network_data'][net_type][ + 'dhcp_end'] = dhcp_end - vlan_network_data[net_type]['static_start'] = static_start - vlan_network_data[net_type]['static_end'] = static_end + self.data['network']['vlan_network_data'][net_type][ + 'static_start'] = static_start + self.data['network']['vlan_network_data'][net_type][ + 'static_end'] = static_end # There is no vlan for oob network if (net_type != 'oob'): - vlan_network_data[net_type]['vlan'] = self.data['network'][ - 'vlan_network_data'][net_type]['vlan'] + self.data['network']['vlan_network_data'][net_type][ + 'vlan'] = self.data['network']['vlan_network_data'][ + net_type]['vlan'] # OAM have default routes. Only for cruiser. TBD if (net_type == 'oam'): routes = ["0.0.0.0/0"] else: routes = [] - vlan_network_data[net_type]['routes'] = routes - - # Update network data to self.data - self.data['network']['vlan_network_data'][ - net_type] = vlan_network_data[net_type] + self.data['network']['vlan_network_data'][net_type][ + 'routes'] = routes LOG.debug("Updated vlan network data:\n{}".format( - pprint.pformat(vlan_network_data))) + pprint.pformat(self.data['network']['vlan_network_data']))) def load_extracted_data_from_data_source(self, extracted_data): """ @@ -259,22 +321,22 @@ def load_extracted_data_from_data_source(self, extracted_data): extracted_data = yaml.safe_load(raw_data) """ - LOG.info("Load extracted data from data source") - self._validate_extracted_data(extracted_data) + LOG.info("Loading plugin data source") self.data = extracted_data - LOG.debug("Extracted data from plugin data source:\n{}".format( + LOG.debug("Extracted data from plugin:\n{}".format( pprint.pformat(extracted_data))) extracted_file = "extracted_file.yaml" yaml_file = yaml.dump(extracted_data, default_flow_style=False) with open(extracted_file, 'w') as f: f.write(yaml_file) f.close() + # Append region_data supplied from CLI to self.data self.data['region_name'] = self.region_name def dump_intermediary_file(self, intermediary_dir): - """ Dumping intermediary yaml """ - LOG.info("Dumping intermediary yaml") + """ Writing intermediary yaml """ + LOG.info("Writing intermediary yaml") intermediary_file = "{}_intermediary.yaml".format( self.data['region_name']) # Check of if output dir = intermediary_dir exists @@ -282,7 +344,7 @@ def dump_intermediary_file(self, intermediary_dir): outfile = "{}/{}".format(intermediary_dir, intermediary_file) else: outfile = intermediary_file - LOG.info("Intermediary file dir:{}".format(outfile)) + LOG.info("Intermediary file:{}".format(outfile)) yaml_file = yaml.dump(self.data, default_flow_style=False) with open(outfile, 'w') as f: f.write(yaml_file) @@ -290,11 +352,11 @@ def dump_intermediary_file(self, intermediary_dir): def generate_intermediary_yaml(self): """ Generating intermediary yaml """ - LOG.info("Generating intermediary yaml") + LOG.info("Start: Generate Intermediary") self._apply_design_rules() self._get_genesis_node_details() + self._validate_intermediary_data(self.data) self.intermediary_yaml = self.data - # TODO(pg710r):self._modify_intermediary() return self.intermediary_yaml def edit_intermediary_yaml(self): diff --git a/spyglass/sample/SiteDesignSpec_v0.1.xlsx b/spyglass/sample/SiteDesignSpec_v0.1.xlsx new file mode 100644 index 0000000..cdf8278 Binary files /dev/null and b/spyglass/sample/SiteDesignSpec_v0.1.xlsx differ diff --git a/spyglass/sample/excel_spec.yaml b/spyglass/sample/excel_spec.yaml new file mode 100644 index 0000000..62831a0 --- /dev/null +++ b/spyglass/sample/excel_spec.yaml @@ -0,0 +1,63 @@ +# Copyright 2018 The Openstack-Helm Authors. +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Important: Please modify the dictionary with appropriate +# design spec file. +--- +specs: + # Design Spec file name: SiteDesignSpec_v0.1.xlsx + xl_spec: + ipmi_sheet_name: 'Site-Information' + start_row: 4 + end_row: 15 + hostname_col: 2 + ipmi_address_col: 3 + host_profile_col: 5 + ipmi_gateway_col: 4 + private_ip_sheet: 'Site-Information' + net_type_col: 1 + vlan_col: 2 + vlan_start_row: 19 + vlan_end_row: 30 + net_start_row: 33 + net_end_row: 40 + net_col: 2 + net_vlan_col: 1 + public_ip_sheet: 'Site-Information' + oam_vlan_col: 1 + oam_ip_row: 43 + oam_ip_col: 2 + oob_net_row: 48 + oob_net_start_col: 2 + oob_net_end_col: 5 + ingress_ip_row: 45 + dns_ntp_ldap_sheet: 'Site-Information' + login_domain_row: 52 + ldap_col: 2 + global_group: 53 + ldap_search_url_row: 54 + ntp_row: 55 + ntp_col: 2 + dns_row: 56 + dns_col: 2 + domain_row: 51 + domain_col: 2 + location_sheet: 'Site-Information' + column: 2 + corridor_row: 59 + site_name_row: 58 + state_name_row: 60 + country_name_row: 61 + clli_name_row: 62 diff --git a/spyglass/sample/site_config.yaml b/spyglass/sample/site_config.yaml new file mode 100644 index 0000000..25fa990 --- /dev/null +++ b/spyglass/sample/site_config.yaml @@ -0,0 +1,33 @@ +################################## +# Site Specific Tugboat Settings # +################################## +--- +site_info: + ldap: + common_name: test + url: ldap://ldap.example.com + subdomain: test + ntp: + servers: 10.10.10.10,20.20.20.20,30.30.30.30 + sitetype: foundry + domain: atlantafoundry.com + dns: + servers: 8.8.8.8,8.8.4.4,208.67.222.222 +network: + vlan_network_data: + ingress: + subnet: + - 132.68.226.72/29 + bgp : + peers: + - '172.29.0.2' + - '172.29.0.3' + asnumber: 64671 + peer_asnumber: 64688 +storage: + ceph: + controller: + osd_count: 6 +... + + diff --git a/spyglass/schemas/data_schema.json b/spyglass/schemas/data_schema.json index 53182cc..7be761f 100644 --- a/spyglass/schemas/data_schema.json +++ b/spyglass/schemas/data_schema.json @@ -140,8 +140,11 @@ "properties": { "subnet": { "description": "Subnet address of the network", - "type": "string", - "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + "type": "array", + "items": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } }, "vlan": { "description": "Vlan id of the network", @@ -166,13 +169,8 @@ "pattern":"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" } ] + } }, - "vlan": { - "description": "Vlan id of the network", - "type": "string", - "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])$" - } - }, "required": [ "subnet" ] @@ -182,8 +180,11 @@ "properties": { "subnet": { "description": "Subnet address of the network", + "type": "array", + "items": { "type": "string", "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } }, "vlan": { "description": "Vlan id of the network", @@ -201,18 +202,20 @@ "properties": { "subnet": { "description": "Subnet address of the network", + "type": "array", + "items": { "type": "string", "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } }, "vlan": { "description": "Vlan id of the network", "type": "string", - "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])$" + "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])?$" } }, "required": [ - "subnet", - "vlan" + "subnet" ] }, "pxe": { @@ -220,8 +223,11 @@ "properties": { "subnet": { "description": "Subnet address of the network", + "type": "array", + "items": { "type": "string", "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } }, "vlan": { "description": "Vlan id of the network", @@ -239,8 +245,11 @@ "properties": { "subnet": { "description": "Subnet address of the network", + "type": "array", + "items": { "type": "string", "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } }, "vlan": { "description": "Vlan id of the network", @@ -252,8 +261,7 @@ "subnet", "vlan" ] - } - + } }, "required" :[ "calico", diff --git a/spyglass/spyglass.py b/spyglass/spyglass.py index 46b15fc..2bc19d1 100644 --- a/spyglass/spyglass.py +++ b/spyglass/spyglass.py @@ -69,7 +69,7 @@ def generate_manifest_files(intermediary, manifest_dir=None): @click.option( '--edit_intermediary/--no_edit_intermediary', '-e/-nedit', - default=True, + default=False, help='Flag to let user edit intermediary') @click.option( '--generate_manifests', @@ -81,6 +81,18 @@ def generate_manifest_files(intermediary, manifest_dir=None): '-mdir', type=click.Path(exists=True), help='The path where manifest files needs to be generated') +@click.option( + '--excel', + '-x', + multiple=True, + type=click.Path(exists=True), + help= + 'Path to engineering excel file, to be passed with generate_intermediary') +@click.option( + '--excel_spec', + '-e', + type=click.Path(exists=True), + help='Path to excel spec, to be passed with generate_intermediary') @click.option( '--loglevel', '-l', @@ -108,7 +120,7 @@ def main(*args, **kwargs): stream_handle.setFormatter(formatter) LOG.addHandler(stream_handle) LOG.info("Spyglass start") - LOG.debug("CLI Parameters passed:\n{}".format(kwargs)) + LOG.info("CLI Parameters passed:\n{}".format(kwargs)) # When intermediary file is specified, Spyglass will generate the # manifest without extracting any data from plugin data source @@ -135,14 +147,13 @@ def main(*args, **kwargs): raw_data = config.read() additional_config_data = yaml.safe_load(raw_data) - LOG.debug("Additional config data:\n{}".format( + LOG.debug("Additional site config data passed:\n{}".format( pprint.pformat(additional_config_data))) data_extractor.set_config_opts(plugin_conf) data_extractor.extract_data() LOG.info( "Apply additional configuration from:{}".format(additional_config)) data_extractor.apply_additional_data(additional_config_data) - LOG.debug(pprint.pformat(data_extractor.site_data)) """ Initialize ProcessDataSource object to process received data """