diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8ff5747 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +lg-logs/* \ No newline at end of file diff --git a/BgpParser.py b/BgpParser.py index bae1064..33f7fb3 100644 --- a/BgpParser.py +++ b/BgpParser.py @@ -1,5 +1,6 @@ __author__ = "Vasilis Giotsas" __email__ = "" + # This software is Copyright (C) 2015 The Regents of the University of # California. All Rights Reserved. Permission to copy, modify, and # distribute this software and its documentation for educational, research @@ -22,7 +23,7 @@ # # This software program and documentation are copyrighted by The Regents of # the University of California. The software program and documentation are -# supplied “as is”, without any accompanying services from The Regents. The +# supplied "as is", without any accompanying services from The Regents. The # Regents does not warrant that the operation of the program will be # uninterrupted or error-free. The end-user understands that the program # was developed for research purposes and is advised not to rely @@ -35,14 +36,15 @@ # THE POSSIBILITY OF SUCH DAMAGE. THE UNIVERSITY OF CALIFORNIA SPECIFICALLY # DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE -# SOFTWARE PROVIDED HEREUNDER IS ON AN “AS IS” BASIS, AND THE UNIVERSITY OF +# SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF # CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, # ENHANCEMENTS, OR MODIFICATIONS. -from Helper import Helper + import re +import ujson +import logging import operator -from math import ceil -import sys +from Helper import Helper class Prefix: @@ -62,13 +64,23 @@ class BgpParser: """ Methods to parse BGP output from looking glasses queries and extract BGP attributes """ + + def __init__(self): + + # for each AS maps which of the other ASes colocated in the same + # Route Server are allowed to receive its prefix advertisments + self.allowed = {} + # stores if an AS has set the block_all policy for a prefix + # this is necessary for BGP records in which the communities attribute + # spans multiple lines + self.block_all = set() + @staticmethod def samplePrefixes(asnToPrfxCount, prfxList, ixp, outfile): """ This function selects a sample of the advertised prefixes to minimize the number of queries to the looking glasses (see section 4.3 of the paper "Inferring multilateral peering". """ - filename = "" prefixesToQuery = set() # the list of prefixes to query asnQueryPrefixes = {} # the set that holds the number of prefixes to be queried for each ASN # iterate prefixes sorted by the number of ASes that advertise each prefix @@ -83,15 +95,26 @@ def samplePrefixes(asnToPrfxCount, prfxList, ixp, outfile): asnQueryPrefixes[asn] += 1 if query_this_prefix: prefixesToQuery.add(prefix.prefix) - filename = Helper.saveToFile(outfile, prefix.prefix+"\n", "a+", ixp) + Helper.saveToFile(outfile, prefix.prefix + "\n", "a+", ixp) print "Use " + str(len(prefixesToQuery)) + " out of " + str(len(prfxList)) return prefixesToQuery @staticmethod - def parseNeighInfo(inputfile, ipToAsn, ixp, outfile): + def parse_neigh_info(inputfile, ipToAsn, ixp, outfile): """ Function to parse a BGP neighbor output file. - It returns a subset of the advertised prefixes to be queried for BGP info + It returns a subset of the advertised prefixes to be queried for BGP info. + + :param inputfile: the path to the file that contains the output of the BGP neighbor command + :type inputfile: string + :param ipToAsn: the mapping between ASNs and next-hop IP addresses + :type ipToAsn: dict + :param ixp: the ASN of the IXP Route Server + :type ixp: string + :param outfile: the path to the output file + :type outfile: string + :return: a set of IP addresses to query using the `show ip bgp` command + :rtype: set """ print "Parsing neighbor file", inputfile asnToPrfxCount = {} @@ -115,8 +138,14 @@ def parseNeighInfo(inputfile, ipToAsn, ixp, outfile): else: # If the line contains an IP prefix and starts with '*' it means that I reached the BGP table and # therefore I proceed to extract the prefixes for the corresponding ASN - prefix = re.findall(r'(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3}\/(?:[\d]{2}))', - line) + prefix_v4 = re.findall(r'(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3}\/(?:[\d]{2}))', + line) + prefix_v6 = re.search(r'((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?', line) + if prefix_v6 is not None: + prefix_v6 = [prefix_v6.group()] + else: + prefix_v6 = [] + prefix = prefix_v4 + prefix_v6 if prefix is not None and len(prefix) > 0: if line.startswith("*") or line.startswith(prefix[0]): asnToPrfxCount[asn] += 1 @@ -126,9 +155,34 @@ def parseNeighInfo(inputfile, ipToAsn, ixp, outfile): prfxList[prefix[0]].addToAsnList(asn) return BgpParser.samplePrefixes(asnToPrfxCount, prfxList, ixp, outfile) + @staticmethod + def parse_summary_alice(input_file): + """ + Parses the summary command JSON output of the Alice LG. + + :param input_file: The path to the file with the BGP summary output. + :type input_file: string + :return: The next-hop IP, ASN, and ID for each route server member + :rtype: dict + """ + ipToAsn = {} + with open(input_file, 'rb') as f: + try: + for line in f: + d = ujson.loads(line.strip()) + for neighbor in d["neighbours"]: + if neighbor["routes_received"] > 0: + ipToAsn[neighbor["id"]] = str(neighbor["asn"]) + " " + neighbor["address"] + break + except ValueError: + logging.error("The JSON format doesn't conform to the Alice LG API schema.") + except KeyError: + logging.error("The JSON format doesn't conform to the Alice LG API schema.") + + return ipToAsn @staticmethod - def parse_summary(ixp, inputfile, ipversion, ixpParam): + def parse_summary(inputfile, ipversion, ixpParam): """ Function to parse a BGP summary output file. It prints the ASN->Neighbor IP mapping in a file @@ -136,245 +190,233 @@ def parse_summary(ixp, inputfile, ipversion, ixpParam): ipToAsn = {} addrPos, asnPos, ipcountPos, rtrType = [int(ixpParam["summary"]["ip"]), int(ixpParam["summary"]["asn"]), int(ixpParam["summary"]["ipCount"]), ixpParam["type"]] - with open(inputfile, 'rb') as f: - for line in f: - # split the line to white spaces - lineTokens = line.strip().split() - if len(lineTokens) <= ipcountPos: continue - interfaces = re.findall( - r'(?:\s|^|\(|\[)(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})(?:\s|\)|$|\])', line) - if len(lineTokens) > addrPos and len(interfaces) > 0: - # check if the string that is supposed to be in the position of the address is indeed a valid IP address - ip = lineTokens[addrPos] - ipType = Helper.getIPNetwork(ip) - if str(ipType) == str(ipversion) or (ipType > 0 and int(ipversion) == 10): - # check if the string in the position of the ASN is a valid number - asn = lineTokens[asnPos] - asn = asn.replace("AS", "") - if '.' in asn: - asn = Helper.convertToAsn32(asn) - if Helper.isPositiveInt(asn): - # check if the ASN is active and advertises prefixes - # often the number of advertised prefixes may be split in total received/best - # in this case we want the total which is the first of the two numbers - ipcount = lineTokens[ipcountPos] - try: - if rtrType == "bird": - received = re.findall(r"[\w']+", ipcount)[0] - elif rtrType == "quagga": - received = ipcount - except: - print ipcount - # if string represents a valid positive number add asn->ip mapping to the dictionary - if Helper.isPositiveInt(received): - if ip not in ipToAsn: - ipToAsn[ip] = asn + if rtrType == "alice": + ipToAsn = BgpParser.parse_summary_alice(inputfile) + else: + with open(inputfile, 'rb') as f: + for line in f: + # split the line to white spaces + lineTokens = line.strip().split() + if len(lineTokens) <= ipcountPos: + continue + interfaces_v4 = re.findall( + r'(?:\s|^|\(|\[)(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})(?:\s|\)|$|\])', line) + interfaces_v6 = re.search(r'((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?', line) + + if interfaces_v6 is not None: + v6_interfaces = [interfaces_v6.group()] + else: + v6_interfaces = [] + interfaces = interfaces_v4 + v6_interfaces + if len(lineTokens) > addrPos and len(interfaces) > 0: + # check if the string that is supposed to be in the position of the address is indeed a valid IP address + ip = lineTokens[addrPos] + ipType = Helper.getIPNetwork(ip) + if str(ipType) == str(ipversion) or (ipType > 0 and int(ipversion) == 10): + # check if the string in the position of the ASN is a valid number + asn = lineTokens[asnPos] + asn = asn.replace("AS", "") + if '.' in asn: + asn = Helper.convertToAsn32(asn) + if Helper.isPositiveInt(asn): + # check if the ASN is active and advertises prefixes + # often the number of advertised prefixes may be split in total received/best + # in this case we want the total which is the first of the two numbers + ipcount = lineTokens[ipcountPos] + try: + if rtrType == "bird": + received = re.findall(r"[\w']+", ipcount)[0] + elif rtrType == "quagga": + received = ipcount + except IndexError: + logging.error("Could not parse the IP count for line: '%s'\n" % line.strip()) + # if string represents a valid positive number add asn->ip mapping to the dictionary + if Helper.isPositiveInt(received): + if ip not in ipToAsn: + ipToAsn[ip] = asn return ipToAsn - @staticmethod - def parse_prefix_info_quagga(inputfile, ip2asn, rs_community_allow, rs_community_block): + def parse_prefix_communities(self, prefix_communities, rs_communities, rs_members, next_hop, communities_line=1): + """ + Parses the RS communities attached on a prefix + """ + tmp_allowed = set() + tmp_blocked = set() + + allow_all = True # Default configuration of Route Servers + for community in prefix_communities: + if "," in community: + # transform community forma from (x,y) to x:y + community = community[1:-1].replace(",", ":") + first_16 = community.split(":")[0] + bottom_16 = community.split(":")[1] + if community in rs_communities["block_all"]: + allow_all = False + self.block_all.add(next_hop) + tmp_blocked = set(rs_members) + elif community == rs_communities["allow_one"].replace("$AS", first_16): + tmp_allowed.add(first_16) + elif community == rs_communities["allow_one"].replace("$AS", bottom_16): + tmp_allowed.add(bottom_16) + elif community == rs_communities["block_one"].replace("$AS", first_16): + tmp_blocked.add(first_16) + elif community == rs_communities["block_one"].replace("$AS", bottom_16): + tmp_blocked.add(bottom_16) + + # B-IX uses the same community for allow_one and block_one, the community semantics + # depend on whether the block_all community is present or not: + # http://www.b-ix.net/technical/requirements/ + if allow_all is True and "59900:0" in rs_communities["block_all"]: + tmp_blocked |= tmp_allowed + tmp_allowed = set() + + # if no "block all" community was present assume the default route server behaviour + # which is to advertise the received prefixes to every route server member + if allow_all and next_hop not in self.block_all: + if communities_line > 1: + tmp_allowed |= set(self.allowed.get(next_hop, rs_members)) - tmp_blocked + else: + tmp_allowed |= rs_members - tmp_blocked + try: + self.allowed[next_hop].extend(tmp_allowed) + except KeyError: + self.allowed[next_hop] = list(tmp_allowed) + + def calculate_mlp_links(self, encountered_members): + """ + Calculates the multilateral peering links over the Route Server + based on the policies (allowed/blocked) defined by the path + redistribution communities of each Route Server member + """ + links = set() + as_to_link = dict() + for m1 in encountered_members: + # If no community was set for the AS RS member m1, + # we assume the "Allow all" default RS policy + if m1 not in self.allowed: + self.allowed[m1] = encountered_members + for m2 in self.allowed[m1]: + if m2 in encountered_members: + # If no community was set for the AS RS member m2, + # we assume the "Allow all" default RS policy + if m2 not in self.allowed: + self.allowed[m2] = encountered_members + if m1 in self.allowed[m2] and m1 != m2: + link = m1 + " " + m2 + if int(m1) > int(m2): + link = m2 + " " + m1 + links.add(link) + if m1 not in as_to_link: + as_to_link[m1] = set() + if m2 not in as_to_link: + as_to_link[m2] = set() + as_to_link[m1].add(m2) + as_to_link[m2].add(m1) + print len(encountered_members) + for asn in as_to_link: + print asn, " ", len(as_to_link[asn]) + return links + + def parse_prefix_info_quagga(self, inputfile, ip2asn, rs_communities): """ - Function that parses the results of the `show ip bgp` command and combines them with the - route server community files and list of route server members to infer the multilateral + Function that parses the results of the `show ip bgp` command and combines them with the + route server community files and list of route server members to infer the multilateral peering links established over the route server. """ encountered_members = set() - allowed = dict() - blocked = dict() members = set(ip2asn.values()) - # initialize the allowed and blocked lists - for member in members: - allowed[member] = list() - allowed[member].extend(members) # this is the default RS behaviour, which can be overriden by communities - for member in members: - blocked[member] = list() - flag = 0 next_hop = "" - tmp_allowed = set() - tmp_blocked = set() + previous_line = "" + community_pattern = r"(? 0: - for asn in tmp_blocked: - if next_hop in allowed and asn in allowed[next_hop]: - allowed[next_hop].remove(asn) - for asn in tmp_allowed: - if next_hop in allowed: - allowed[next_hop].append(asn) - if line.startswith("#>"): #reset + elif line.startswith("Nexthop") and "RS-client" not in previous_line: + as_path = previous_line.strip().split() + next_hop = as_path[0] + if next_hop in members: + encountered_members.add(next_hop) + elif line.startswith("Communit"): + communities = re.findall(community_pattern, line) + self.parse_prefix_communities(communities, rs_communities, members, next_hop) + elif line.startswith("#>"): # reset next_hop = "" - tmp_allowed = set() - tmp_blocked = set() - links = set() - as_to_link = dict() - for m1 in encountered_members: - for m2 in allowed[m1]: - if m1 in allowed[m2] and m1 != m2: - link = m1 + " " + m2 - if int(m1) > int(m2): - link = m2 + " " + m1 - links.add(link) - if m1 not in as_to_link: - as_to_link[m1] = set() - if m2 not in as_to_link: - as_to_link[m2] = set() - as_to_link[m1].add(m2) - as_to_link[m2].add(m1) - print len(encountered_members) - for asn in as_to_link: - print asn," ",len(as_to_link[asn]) - return links + previous_line = line - @staticmethod - def parse_prefix_info(inputfile, ip2asn, rs_communities): + return self.calculate_mlp_links(encountered_members) + + def parse_prefix_info_bird(self, inputfile, ip2asn, rs_communities): """ Function that parses the results of the `show ip bgp` command and combines them with the route server community files and list of route server members to infer the multilateral peering links established over the route server. """ encountered_members = set() - allowed = dict() - blocked = dict() members = set(ip2asn.values()) - # initialize the allowed and blocked lists - for member in members: - allowed[member] = list() - allowed[member].extend(members) # this is the default RS behaviour, which can be overriden by communities - for member in members: - blocked[member] = list() - flag = 0 + next_hop = "" - tmp_allowed = set() - tmp_blocked = set() + communities_line = 0 + community_pattern = r"\(\d+,\d+\)" with open(inputfile, 'rb') as fin: for line in fin: line = line.strip() - if line.startswith("BGP.as_path"): + communities = re.findall(community_pattern, line) + if len(communities) > 0: + communities_line += 1 + self.parse_prefix_communities(communities, rs_communities, members, next_hop, communities_line) + elif line.startswith("BGP.as_path"): + # clear the block_all set for each new prefix + self.block_all.clear() + communities_line = 0 line_tokens = line.split(":") - as_path = line_tokens[1].strip().split() - next_hop = as_path[0] + asn_path = line_tokens[1].strip().split() + next_hop = asn_path[0] if next_hop in members: encountered_members.add(next_hop) - if line.startswith("BGP.community"): - line_tokens = line.split(":") - communities = line_tokens[1].strip().split() - - # Do two passes, one for "all or nothing" communities, and one for communities - # that allow or block specific ASes - - # 1st pass - aon_community = False - for c in communities: - community = c[1:-1] # remove the parentheses - first_16 = community.split(",")[0] - last_16 = community.split(",")[1] - # check for no-export or no-advertise communities according to rfc1997 - block_all = (first_16 == rs_community_allow or first_16 == "65535") and (last_16 == "65281" or last_16 == "65282") - if community in rs_communities["block_all"] or block_all: - tmp_blocked |= members - tmp_allowed = set() - aon_community = True - elif community in rs_communities["allow_all"]: - tmp_allowed |= members - tmp_blocked = set() - aon_community = True - - # if no "all or nothing" (aon) community was present assume the default route server behaviour - # which is to advertise the received prefixes to every route server member - if aon_community is False: - tmp_allowed |= members - tmp_blocked = set() - - # 2nd pass - for c in communities: - community = c[1:-1] # remove the parentheses - first_16 = community.split(",")[0] - last_16 = community.split(",")[1] - if first_16 == rs_community_block and last_16 != rs_community_allow: - if last_16 in members: - tmp_blocked.add(last_16) - if last_16 in tmp_allowed: - tmp_allowed.remove(last_16) - elif first_16 == rs_community_allow and last_16 != rs_community_allow: - if last_16 in members: - tmp_allowed.add(last_16) - if last_16 in tmp_blocked: tmp_blocked.remove(last_16) - if len(tmp_blocked) > 0: - for asn in tmp_blocked: - if next_hop in allowed and asn in allowed[next_hop]: - allowed[next_hop].remove(asn) - for asn in tmp_allowed: - if next_hop in allowed: - allowed[next_hop].append(asn) - if line.startswith("#>"): #reset + elif line.startswith("#>"): # reset next_hop = "" - tmp_allowed = set() - tmp_blocked = set() - links = set() - as_to_link = dict() - for m1 in encountered_members: - for m2 in allowed[m1]: - if m1 in allowed[m2] and m1 != m2: - link = m1 + " " + m2 - if int(m1) > int(m2): - link = m2 + " " + m1 - links.add(link) - if m1 not in as_to_link: - as_to_link[m1] = set() - if m2 not in as_to_link: - as_to_link[m2] = set() - as_to_link[m1].add(m2) - as_to_link[m2].add(m1) - print len(encountered_members) - for asn in as_to_link: - print asn," ",len(as_to_link[asn]) - return links + + return self.calculate_mlp_links(encountered_members) + + def parse_prefix_info_alice(self, input_file, ip2asn, rs_communities): + """ + Parses the prefix information in the output of the neighbor/$id/routes + endpoint of the Alice LG, to infer the multilateral peering links established + over the route server. + + :param input_file: the path to the file that contains the output of the neighbor/$id/routes endpoint + :type input_file: string + :param ip2asn: the mapping between neighbor IDs and ASNs + :type ip2asn: dict + :param rs_communities: the route redistribution communities used by the route server + :param: dict + :return: the inferred multilateral peering links + :rtype: set + """ + encountered_members = set() + members = set(ip2asn.values()) + with open(input_file) as fin: + for line in fin: + try: + neighbor_prefixes = ujson.loads(line.strip()) + for prefix_data in neighbor_prefixes["imported"]: + next_hop = str(prefix_data["bgp"]["as_path"][0]) + if next_hop in members: + encountered_members.add(next_hop) + else: + print next_hop + communities = list() + for c in prefix_data["bgp"]["communities"]: + communities.append("%s:%s" % (c[0], c[1])) + self.parse_prefix_communities(communities, rs_communities, members, next_hop, 1) + except ValueError: + continue + except KeyError: + continue + return self.calculate_mlp_links(encountered_members) diff --git a/Helper.py b/Helper.py index 420c624..b05b033 100644 --- a/Helper.py +++ b/Helper.py @@ -1,5 +1,6 @@ __author__ = "Vasilis Giotsas" __email__ = "" + # This software is Copyright (C) 2015 The Regents of the University of # California. All Rights Reserved. Permission to copy, modify, and # distribute this software and its documentation for educational, research @@ -22,7 +23,7 @@ # # This software program and documentation are copyrighted by The Regents of # the University of California. The software program and documentation are -# supplied “as is”, without any accompanying services from The Regents. The +# supplied "as is", without any accompanying services from The Regents. The # Regents does not warrant that the operation of the program will be # uninterrupted or error-free. The end-user understands that the program # was developed for research purposes and is advised not to rely @@ -35,18 +36,19 @@ # THE POSSIBILITY OF SUCH DAMAGE. THE UNIVERSITY OF CALIFORNIA SPECIFICALLY # DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE -# SOFTWARE PROVIDED HEREUNDER IS ON AN “AS IS” BASIS, AND THE UNIVERSITY OF +# SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF # CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, # ENHANCEMENTS, OR MODIFICATIONS. -from netaddr import * import os +from netaddr import * + class Helper: """ Collection of helper classes used throughout the program """ - + @staticmethod def getIPNetwork(ipnet): """ diff --git a/HttpQueryHandler.py b/HttpQueryHandler.py index 6a14a30..d4c5307 100644 --- a/HttpQueryHandler.py +++ b/HttpQueryHandler.py @@ -1,5 +1,7 @@ + __author__ = "Vasilis Giotsas" __email__ = "" + # This software is Copyright (C) 2015 The Regents of the University of # California. All Rights Reserved. Permission to copy, modify, and # distribute this software and its documentation for educational, research @@ -22,7 +24,7 @@ # # This software program and documentation are copyrighted by The Regents of # the University of California. The software program and documentation are -# supplied $B!H(Bas is$B!I(B, without any accompanying services from The Regents. The +# supplied "as is", without any accompanying services from The Regents. The # Regents does not warrant that the operation of the program will be # uninterrupted or error-free. The end-user understands that the program # was developed for research purposes and is advised not to rely @@ -35,18 +37,20 @@ # THE POSSIBILITY OF SUCH DAMAGE. THE UNIVERSITY OF CALIFORNIA SPECIFICALLY # DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE -# SOFTWARE PROVIDED HEREUNDER IS ON AN $B!H(BAS IS$B!I(B BASIS, AND THE UNIVERSITY OF +# SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF # CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, # ENHANCEMENTS, OR MODIFICATIONS. -import urllib, urllib2, time, re -from bs4 import BeautifulSoup +import urllib +import urllib2 +import time +import re +from bs4 import BeautifulSoup from HTMLParser import HTMLParser from pprint import pprint import sys import json import httplib -import random import socket from StringIO import StringIO import pycurl @@ -59,11 +63,14 @@ class MLStripper(HTMLParser): """ Methods to strip html tags from string """ + def __init__(self): self.reset() self.fed = [] + def handle_data(self, d): self.fed.append(d) + def get_data(self): return ' '.join(self.fed) @@ -72,6 +79,7 @@ class HttpQueryHandler(object): """ Methods to handle issue of HTTP queries and parsing of the responses """ + def __init__(self): pass @@ -84,12 +92,12 @@ def strip_tags(html): @staticmethod def remove_break_line(html_txt): result = "" - tokens = re.split("
|
|
",html_txt) + tokens = re.split("
|
|
", html_txt) for token in tokens: result = result + token.strip() + "\n" - result = result.replace(" ","") + result = result.replace(" ", "") return result - + @staticmethod def send_http_request(params, url, referrer, verb): user_agent = 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:17.0) Gecko/20100101 Firefox/17' @@ -105,9 +113,9 @@ def send_http_request(params, url, referrer, verb): if url == "http://noc.ucomline.net/lg1": headers.append("Cookie: PHPSESSID=199d19597747f15a4d778d68b56d4ab7") post_fields = urllib.urlencode(params) - pprint( post_fields) + pprint(post_fields) curl_connector.setopt(pycurl.HTTPHEADER, headers) - curl_connector.setopt(curl_connector.URL, url) + curl_connector.setopt(curl_connector.URL, url) curl_connector.setopt(curl_connector.POSTFIELDS, post_fields) elif verb == "get": headers = ['User-Agent: ' + user_agent, @@ -116,7 +124,7 @@ def send_http_request(params, url, referrer, verb): 'Cache-Control: max-age=0'] url += "?" counter = 1 - for key,value in params.items(): + for key, value in params.items(): counter += 1 if key == "zeroparam": url += value @@ -138,12 +146,11 @@ def send_http_request(params, url, referrer, verb): content = storage.getvalue() except pycurl.error, e: error_code, error_text = e.args - print 'We got an error. Code: %s, Text:%s'%(error_code, error_text) + print 'We got an error. Code: %s, Text:%s' % (error_code, error_text) if str(error_code) != "18": return -1 else: content = storage.getvalue() - #print content return content @staticmethod @@ -172,15 +179,18 @@ def send_http_request_alt(params, url, referrer, verb): 'Connection': 'keep-alive'} url += "?" counter = 1 - for key,value in params.items(): + for key, value in params.items(): counter += 1 if key == "zeroparam": url += value + elif key == "lg_query_arg": + continue else: url += key + "=" + value - if counter <= len(params): + if counter <= len(params) - 1: url += "&" url = url.replace(" ", "%20") + print url req = urllib2.Request(url, None, headers) response = urllib2.urlopen(req, timeout=timeout) try: @@ -203,7 +213,7 @@ def send_http_request_alt(params, url, referrer, verb): return the_page @classmethod - def parse_html(cls, html_text, lg_html): + def parse_html(cls, html_text, lg_html, asn): """ Parse the HTML of the reply to extract the LG output """ @@ -215,24 +225,47 @@ def parse_html(cls, html_text, lg_html): insert_br = lg_html["insertbr"] contents = "" try: + soup = BeautifulSoup(html_text, "html.parser") if html_tag == "result": - soup = BeautifulSoup(html_text) pre = soup.findAll("div", {"class": "result"}) pre = pre[1] reply = ''.join(str(pre.contents)) - contents = BeautifulSoup(reply) + contents = BeautifulSoup(reply, "html.parser") elif html_tag == "json": + decoded_html = None decoded = json.loads(html_text) if 'lg' in decoded: contents = decoded['lg'] + elif 'title' in decoded and "rs1.ripn.net" in decoded['title']: + if 'txt' in decoded: + if "BGP information for neighbor" in decoded['txt'][0]: + decoded_html = '\n'.join(decoded['txt'][14:]) + else: + decoded_html = '\n'.join(decoded['txt']) + sanitized_html = decoded_html.replace("", "\t") + soup = BeautifulSoup(sanitized_html, "html.parser") + # html_pretty = soup.prettify() + # soup = BeautifulSoup(html_pretty, "html.parser") + return soup.get_text() + else: + contents = decoded + elif html_tag == "table": + if html_id != "": + table_rows = soup.find("table", {"id": html_id}).find_all('tr') + else: + table_rows = soup.find_all("table")[tag_index].find_all('tr') + for tr in table_rows: + if len(contents) == 0: + contents += tr.text.replace("\n", "\t") + else: + contents += "\n" + tr.text.replace("\n", "\t") elif tag_index == -1: contents = cls.remove_break_line(html_text) elif tag_index == -2: contents = html_text else: - soup = BeautifulSoup(html_text) html_pretty = soup.prettify() - soup = BeautifulSoup(html_pretty) + soup = BeautifulSoup(html_pretty, "html.parser") try: if html_class == "" and html_id == "": pre = soup.findAll(html_tag)[tag_index] @@ -250,10 +283,16 @@ def parse_html(cls, html_text, lg_html): contents += "\n" + str(content.strip()) else: contents += str(content.strip()) - contents = BeautifulSoup(contents) + contents = BeautifulSoup(contents, "html.parser") contents = cls.remove_break_line(contents.encode('utf-8').strip()) else: - if len(pre.contents) > content_index: + if asn == "59900": + pre = soup.find_all(html_tag)[tag_index] + pre = str(pre).replace("
", "\n") + pre = str(pre).replace("\r", "\n") + soup = BeautifulSoup(pre, 'html.parser') + contents = soup.text + elif len(pre.contents) > content_index: contents = pre.contents[content_index] contents = cls.remove_break_line(contents.encode('utf-8').strip()) except AttributeError: @@ -261,7 +300,6 @@ def parse_html(cls, html_text, lg_html): # specific cases if html_tag == "table" and tag_index == 0: - contents = contents.replace("","\n") - contents = contents.replace("","") + contents = contents.replace("", "\n") + contents = contents.replace("", "") return contents - diff --git a/ixp_parameters_new.json b/ixp_parameters_new.json index 11d65cf..2072019 100644 --- a/ixp_parameters_new.json +++ b/ixp_parameters_new.json @@ -264,6 +264,12 @@ "ip": 0, "ipCount": 4 }, + "communities": { + "allow_all": "15669:15669", + "block:all": ["0:15669"], + "allow_one": "15669:$AS", + "block_one": "0:$AS" + }, "type": "bird" } }, @@ -938,28 +944,28 @@ "42403": { "commands": { "bgp": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "bgp", - "rtr": "nsk-rs1.ripn.net" + "query":"bgp", + "rtr":"nsk-rs1.ripn.net", + "addr":"$", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" }, "neighbor": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "ne", - "rtr": "nsk-rs1.ripn.net" - }, - "summary": { - "Submit": "1", - "addr": "$", - "community": "", - "lg_query_arg": "ne_arg", - "query": "summary", - "rtr": "nsk-rs1.ripn.net" + "query":"ne", + "rtr":"nsk-rs1.ripn.net", + "addr":"", + "ne_arg":"$", + "community":"", + "lg_query_arg": "ne_arg" + }, + "summary": { + "query":"summary", + "rtr":"nsk-rs1.ripn.net", + "addr":"", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" } }, "html": { @@ -968,24 +974,30 @@ "id": "", "insertbr": false, "striphtml": false, - "tag": "pre", + "tag": "json", "tagindex": 0 }, "http": { "prefix": true, "rarg": "rtr", - "referer": "http://www.ix.ru/eng/nsk/lookingglass.html", - "type": "post", - "url": "http://www.ix.ru/eng/nsk/lookingglass.html" + "referer": "https://www.msk-ix.ru/en/lookingglass?query=", + "type": "get", + "url": "https://www.msk-ix.ru/ajax/lookingglass/" }, - "lat": 54.969977, - "lon": 82.9494049, + "lat": 55.80758, + "lon": 3746679, "output": { "summary": { "asn": 2, "ip": 0, "ipCount": 7 }, + "communities": { + "allow_all":["42403:42403"], + "block_all":["0:42403"], + "allow_one": "42403:$AS", + "block_one": "0:$AS" + }, "type": "bird" } }, @@ -1031,7 +1043,12 @@ "ip": 0, "ipCount": 6 }, - "type": "bird" + "communities": { + "block_all": ["42476:65535", "65535:65281", "65535:65282"], + "block_one": "42476:$AS", + "allow_one": "$AS:42476" + }, + "type": "quagga" } }, "43100": { @@ -1088,28 +1105,28 @@ "43213": { "commands": { "bgp": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "bgp", - "rtr": "ekt-rs1.ripn.net" + "query":"bgp", + "rtr":"ekt-rs1.ripn.net", + "addr":"$", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" }, "neighbor": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "ne", - "rtr": "ekt-rs1.ripn.net" - }, - "summary": { - "Submit": "1", - "addr": "$", - "community": "", - "lg_query_arg": "ne_arg", - "query": "summary", - "rtr": "ekt-rs1.ripn.net" + "query":"ne", + "rtr":"ekt-rs1.ripn.net", + "addr":"", + "ne_arg":"$", + "community":"", + "lg_query_arg": "ne_arg" + }, + "summary": { + "query":"summary", + "rtr":"ekt-rs1.ripn.net", + "addr":"", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" } }, "html": { @@ -1118,58 +1135,58 @@ "id": "", "insertbr": false, "striphtml": false, - "tag": "pre", + "tag": "json", "tagindex": 0 }, "http": { "prefix": true, "rarg": "rtr", - "referer": "http://www.ix.ru/eng/ekt/lookingglass.html", - "type": "post", - "url": "http://www.ix.ru/eng/ekt/lookingglass.html" + "referer": "https://www.msk-ix.ru/en/lookingglass?query=", + "type": "get", + "url": "https://www.msk-ix.ru/ajax/lookingglass/" }, - "lat": 56.813891, - "lon": 60.6549335, + "lat": 55.80758, + "lon": 3746679, "output": { "summary": { "asn": 2, "ip": 0, "ipCount": 7 }, + "communities": { + "allow_all":["43213:43213"], + "block_all":["0:43213"], + "allow_one": "43213:$AS", + "block_one": "0:$AS" + }, "type": "bird" } }, "43690": { "commands": { "bgp": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "bgp", - "rtr": "spb-rs1.ripn.net", - "run.x": "11", - "run.y": "12" + "query":"bgp", + "rtr":"spb-rs1.ripn.net", + "addr":"$", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" }, "neighbor": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "ne", - "rtr": "spb-rs1.ripn.net", - "run.x": "10", - "run.y": "10" - }, - "summary": { - "Submit": "1", - "addr": "$", - "community": "", - "lg_query_arg": "ne_arg", - "query": "summary", - "rtr": "spb-rs1.ripn.net", - "run.x": "26", - "run.y": "23" + "query":"ne", + "rtr":"spb-rs1.ripn.net", + "addr":"", + "ne_arg":"$", + "community":"", + "lg_query_arg": "ne_arg" + }, + "summary": { + "query":"summary", + "rtr":"spb-rs1.ripn.net", + "addr":"", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" } }, "html": { @@ -1178,24 +1195,30 @@ "id": "", "insertbr": false, "striphtml": false, - "tag": "pre", + "tag": "json", "tagindex": 0 }, "http": { "prefix": true, "rarg": "rtr", - "referer": "http://www.ix.ru/eng/spb/lookingglass.html", - "type": "post", - "url": "http://www.ix.ru/eng/spb/lookingglass.html" + "referer": "https://www.msk-ix.ru/en/lookingglass?query=", + "type": "get", + "url": "https://www.msk-ix.ru/ajax/lookingglass/" }, - "lat": 59.9174455, - "lon": 30.3250575, + "lat": 55.80758, + "lon": 3746679, "output": { "summary": { "asn": 2, "ip": 0, "ipCount": 7 }, + "communities": { + "allow_all":["43690:43690"], + "block_all":["0:43690"], + "allow_one": "43690:$AS", + "block_one": "0:$AS" + }, "type": "bird" } }, @@ -1250,28 +1273,28 @@ "47822": { "commands": { "bgp": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "bgp", - "rtr": "smr-rs1.ripn.net" + "query":"bgp", + "rtr":"smr-rs1.ripn.net", + "addr":"$", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" }, "neighbor": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "ne", - "rtr": "smr-rs1.ripn.net" - }, - "summary": { - "Submit": "1", - "addr": "$", - "community": "", - "lg_query_arg": "ne_arg", - "query": "summary", - "rtr": "smr-rs1.ripn.net" + "query":"ne", + "rtr":"smr-rs1.ripn.net", + "addr":"", + "ne_arg":"$", + "community":"", + "lg_query_arg": "ne_arg" + }, + "summary": { + "query":"summary", + "rtr":"smr-rs1.ripn.net", + "addr":"", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" } }, "html": { @@ -1280,52 +1303,58 @@ "id": "", "insertbr": false, "striphtml": false, - "tag": "pre", + "tag": "json", "tagindex": 0 }, "http": { "prefix": true, "rarg": "rtr", - "referer": "http://www.ix.ru/eng/smr/lookingglass.html", - "type": "post", - "url": "http://www.ix.ru/eng/smr/lookingglass.html" + "referer": "https://www.msk-ix.ru/en/lookingglass?query=", + "type": "get", + "url": "https://www.msk-ix.ru/ajax/lookingglass/" }, - "lat": 53.260908, - "lon": 50.198077, + "lat": 55.80758, + "lon": 3746679, "output": { "summary": { "asn": 2, "ip": 0, "ipCount": 7 }, + "communities": { + "allow_all":["47822:47822"], + "block_all":["0:47822"], + "allow_one": "47822:$AS", + "block_one": "0:$AS" + }, "type": "bird" } }, "48216": { "commands": { "bgp": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "bgp", - "rtr": "rnd-rs1.ripn.net" + "query":"bgp", + "rtr":"rnd-rs1.ripn.net", + "addr":"$", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" }, "neighbor": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "ne", - "rtr": "rnd-rs1.ripn.net" - }, - "summary": { - "Submit": "1", - "addr": "$", - "community": "", - "lg_query_arg": "ne_arg", - "query": "summary", - "rtr": "rnd-rs1.ripn.net" + "query":"ne", + "rtr":"rnd-rs1.ripn.net", + "addr":"", + "ne_arg":"$", + "community":"", + "lg_query_arg": "ne_arg" + }, + "summary": { + "query":"summary", + "rtr":"rnd-rs1.ripn.net", + "addr":"", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" } }, "html": { @@ -1334,52 +1363,58 @@ "id": "", "insertbr": false, "striphtml": false, - "tag": "pre", + "tag": "json", "tagindex": 0 }, "http": { "prefix": true, "rarg": "rtr", - "referer": "http://www.ix.ru/eng/rnd/lookingglass.html", - "type": "post", - "url": "http://www.ix.ru/eng/rnd/lookingglass.html" + "referer": "https://www.msk-ix.ru/en/lookingglass?query=", + "type": "get", + "url": "https://www.msk-ix.ru/ajax/lookingglass/" }, - "lat": 47.2610085, - "lon": 39.6279999, + "lat": 55.80758, + "lon": 3746679, "output": { "summary": { "asn": 2, "ip": 0, "ipCount": 7 }, + "communities": { + "allow_all":["48216:48216"], + "block_all":["0:48216"], + "allow_one": "48216:$AS", + "block_one": "0:$AS" + }, "type": "bird" } }, "50706": { "commands": { "bgp": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "bgp", - "rtr": "kzn-rs1.ripn.net" + "query":"bgp", + "rtr":"kzn-rs1.ripn.net", + "addr":"$", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" }, "neighbor": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "ne", - "rtr": "kzn-rs1.ripn.net" - }, - "summary": { - "Submit": "1", - "addr": "$", - "community": "", - "lg_query_arg": "ne_arg", - "query": "summary", - "rtr": "kzn-rs1.ripn.net" + "query":"ne", + "rtr":"kzn-rs1.ripn.net", + "addr":"", + "ne_arg":"$", + "community":"", + "lg_query_arg": "ne_arg" + }, + "summary": { + "query":"summary", + "rtr":"kzn-rs1.ripn.net", + "addr":"", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" } }, "html": { @@ -1388,24 +1423,30 @@ "id": "", "insertbr": false, "striphtml": false, - "tag": "pre", + "tag": "json", "tagindex": 0 }, "http": { "prefix": true, "rarg": "rtr", - "referer": "http://www.ix.ru/eng/kzn/lookingglass.html", - "type": "post", - "url": "http://www.ix.ru/eng/kzn/lookingglass.html" + "referer": "https://www.msk-ix.ru/en/lookingglass?query=", + "type": "get", + "url": "https://www.msk-ix.ru/ajax/lookingglass/" }, - "lat": 55.7955015, - "lon": 49.073303, + "lat": 55.80758, + "lon": 3746679, "output": { "summary": { "asn": 2, "ip": 0, "ipCount": 7 }, + "communities": { + "allow_all":["50706:50706"], + "block_all":["0:50706"], + "allow_one": "50706:$AS", + "block_one": "0:$AS" + }, "type": "bird" } }, @@ -1521,21 +1562,21 @@ "ipv": "4", "lg_query_arg": "addr", "query": "bgp", - "router": "192.168.2.4" + "router": "192.168.2.41" }, - "neighbor": { + "regex": { "addr": "$", "ipv": "4", "lg_query_arg": "addr", - "query": "bgp routes", - "router": "192.168.2.4" + "query": "as routes received", + "router": "192.168.2.41" }, "summary": { "addr": "$", "ipv": "4", "lg_query_arg": "addr", "query": "bgp summary", - "router": "192.168.2.4" + "router": "192.168.2.41" } }, "html": { @@ -1550,17 +1591,22 @@ "http": { "prefix": true, "rarg": "router", - "referer": "http://www.bix.hu/lg/lg.cgi", + "referer": "http://bix.hu/en/looking_glass", "type": "post", - "url": "http://www.bix.hu/lg/lg.cgi" + "url": "http://bix.hu/lg/lg.cgi" }, "lat": 47.4805856, "lon": 19.1303031, "output": { "summary": { - "asn": 2, + "asn": 1, "ip": 0, - "ipCount": 9 + "ipCount": 4 + }, + "communities": { + "block_all": ["5507:10"], + "block_one": "0:$AS:10", + "allow_one": "$AS:1" }, "type": "bird" } @@ -1568,22 +1614,22 @@ "55518": { "commands": { "bgp": { - "argument": "$", - "lg_query_arg": "argument", - "requestid": "15", - "routerid": "10" + "arg": "$", + "lg_query_arg": "arg", + "query": "1", + "router": "Route Server at Global Switch" }, "regex": { - "argument": "$", - "lg_query_arg": "argument", - "requestid": "20", - "routerid": "10" + "arg": "^$", + "lg_query_arg": "arg", + "query": "17", + "router": "Route Server at Global Switch" }, "summary": { - "argument": "", - "lg_query_arg": "argument", - "requestid": "10", - "routerid": "10" + "arg": "", + "lg_query_arg": "arg", + "query": "2", + "router": "Route Server at Global Switch" } }, "html": { @@ -1597,10 +1643,10 @@ }, "http": { "prefix": true, - "rarg": "routerid", + "rarg": "router", "referer": "http://services.sgix.sg/mrlg/lg.php", "type": "post", - "url": "http://services.sgix.sg/mrlg/lg.php" + "url": "http://services.sgix.sg/mrlg" }, "lat": 1.283405, "lon": 103.852196, @@ -1610,34 +1656,39 @@ "ip": 0, "ipCount": 9 }, - "type": "bird" + "communities": { + "block_all": ["0:55518", "65535:65281", "65535:65282"], + "block_one": "0:$AS", + "allow_one": "55518:$AS" + }, + "type": "quagga" } }, "57056": { "commands": { "bgp": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "bgp", - "rtr": "stw-rs1.ripn.net" + "query":"bgp", + "rtr":"stw-rs1.ripn.net", + "addr":"$", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" }, "neighbor": { - "Submit": "1", - "community": "", - "lg_query_arg": "ne_arg", - "ne_arg": "$", - "query": "ne", - "rtr": "stw-rs1.ripn.net" - }, - "summary": { - "Submit": "1", - "addr": "$", - "community": "", - "lg_query_arg": "ne_arg", - "query": "summary", - "rtr": "stw-rs1.ripn.net" + "query":"ne", + "rtr":"stw-rs1.ripn.net", + "addr":"", + "ne_arg":"$", + "community":"", + "lg_query_arg": "ne_arg" + }, + "summary": { + "query":"summary", + "rtr":"stw-rs1.ripn.net", + "addr":"", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" } }, "html": { @@ -1646,66 +1697,89 @@ "id": "", "insertbr": false, "striphtml": false, - "tag": "pre", + "tag": "json", "tagindex": 0 }, "http": { "prefix": true, "rarg": "rtr", - "referer": "http://www.ix.ru/eng/stw/lookingglass.html", - "type": "post", - "url": "http://www.ix.ru/eng/stw/lookingglass.html" + "referer": "https://www.msk-ix.ru/en/lookingglass?query=", + "type": "get", + "url": "https://www.msk-ix.ru/ajax/lookingglass/" }, - "lat": 45.0450234, - "lon": 41.9960869, + "lat": 55.80758, + "lon": 3746679, "output": { "summary": { "asn": 2, "ip": 0, "ipCount": 7 }, + "communities": { + "allow_all":["57056:57056"], + "block_all":["0:57056"], + "allow_one": "57056:$AS", + "block_one": "0:$AS" + }, "type": "bird" } }, "59900": { "commands": { "bgp": { - "addr": "$", - "cmd": "bgp", - "lg_query_arg": "addr", + "n": "$", + "cmd": "sib", + "lg_query_arg": "n", "proto": "ipv4", - "router": "217.174.157.1" + "r": "10.4.4.22" }, "neighbor": { "addr": "$", "cmd": "sibnrr", "lg_query_arg": "addr", "proto": "ipv4", - "router": "217.174.157.1" + "router": "10.4.4.22" }, "summary": { "addr": "$", "cmd": "summary", "lg_query_arg": "addr", "proto": "ipv4", - "router": "217.174.157.1" + "router": "10.4.4.22" } }, - "html": { - "class": "", - "contentindex": -2, - "id": "summary", - "insertbr": false, - "striphtml": true, - "tag": "table", - "tagindex": 0 - }, + "html":[ + { + "class": "", + "contentindex": -1, + "id": "summary", + "insertbr": false, + "striphtml": true, + "tag": "table", + "tagindex": 0, + "commands": ["summary"] + }, + { + "class": "", + "contentindex": 0, + "id": "", + "insertbr": true, + "striphtml": true, + "tag": "pre", + "tagindex": 0, + "commands": ["neighbor", "bgp"] + } + ], "http": { "prefix": true, - "rarg": "router", + "rarg": { + "summary": "router", + "neighbor": "router", + "bgp":"r" + }, "referer": "http://lg.b-ix.net/?cmd=summary&addr=&proto=ipv4&router=217.174.157.1", "type": "get", - "url": "http://lg.b-ix.net" + "url": "http://lg.b-ix.net/" }, "lat": 42.6954322, "lon": 23.3239467, @@ -1715,7 +1789,13 @@ "ip": 0, "ipCount": 10 }, - "type": "bird" + "communities": { + "allow_all": "", + "allow_one": "59900:$AS", + "block_all": ["59900:0"], + "block_one": "59900:$AS" + }, + "type": "quagga" } }, "65000": { @@ -1776,6 +1856,14 @@ "submit": "Submit", "tab": "prefix_info" }, + "bgp_v6": { + "argument": "$", + "lg_query_arg": "argument", + "query": "show ip bgp", + "routeserver": "rs1-active.de-cix.net -6", + "submit": "Submit", + "tab": "prefix_info" + }, "neighbor": { "argument": "$", "lg_query_arg": "argument", @@ -1784,12 +1872,27 @@ "submit": "Submit", "tab": "neighbour_info" }, + "neighbor_v6": { + "argument": "$", + "lg_query_arg": "argument", + "query": "show ip bgp neighbor", + "routeserver": "rs1-active.de-cix.net -6", + "submit": "Submit", + "tab": "neighbour_info" + }, "summary": { "lg_query_arg": "argument", "query": "show ip bgp summary", "routeserver": "rs1-active.de-cix.net", "submit": "Submit", "tab": "bgp_summary" + }, + "summary_v6": { + "lg_query_arg": "argument", + "query": "show ip bgp summary", + "routeserver": "rs1-active.de-cix.net -6", + "submit": "Submit", + "tab": "bgp_summary" } }, "html": { @@ -1816,6 +1919,173 @@ "ip": 0, "ipCount": 5 }, + "communities": { + "block_all": ["0:6695", "6695:65281", "6695:65282", "65535:65281", "65535:65282"], + "block_one": "0:$AS", + "allow_one": "6695:$AS" + }, + "type": "bird" + } + }, + "63034": { + "commands": { + "bgp": { + "argument": "$", + "lg_query_arg": "argument", + "query": "show ip bgp", + "routeserver": "rs1.nyc.de-cix.net", + "submit": "Submit", + "tab": "prefix_info" + }, + "bgp_v6": { + "argument": "$", + "lg_query_arg": "argument", + "query": "show ip bgp", + "routeserver": "rs1.nyc.de-cix.net -6", + "submit": "Submit", + "tab": "prefix_info" + }, + "neighbor": { + "argument": "$", + "lg_query_arg": "argument", + "query": "show ip bgp neighbor", + "routeserver": "rs1.nyc.de-cix.net", + "submit": "Submit", + "tab": "neighbour_info" + }, + "neighbor_v6": { + "argument": "$", + "lg_query_arg": "argument", + "query": "show ip bgp neighbor", + "routeserver": "rs1.nyc.de-cix.net -6", + "submit": "Submit", + "tab": "neighbour_info" + }, + "summary": { + "lg_query_arg": "argument", + "query": "show ip bgp summary", + "routeserver": "rs1.nyc.de-cix.net", + "submit": "Submit", + "tab": "bgp_summary" + }, + "summary_v6": { + "lg_query_arg": "argument", + "query": "show ip bgp summary", + "routeserver": "rs1.nyc.de-cix.net -6", + "submit": "Submit", + "tab": "bgp_summary" + } + }, + "html": { + "class": "", + "contentindex": 0, + "id": "", + "insertbr": false, + "striphtml": false, + "tag": "pre", + "tagindex": 0 + }, + "http": { + "prefix": true, + "rarg": "routeserver", + "referer": "https://lg.nyc.de-cix.net/", + "type": "post", + "url": "https://lg.nyc.de-cix.net/" + }, + "lat": 50.1109, + "lon": 8.6821, + "output": { + "summary": { + "asn": 1, + "ip": 0, + "ipCount": 5 + }, + "communities": { + "block_all": ["0:63034", "63034:65281", "63034:65282", "65535:65281", "65535:65282"], + "block_one": "0:$AS", + "allow_one": "63034:$AS" + }, + "type": "bird" + } + }, + "62499": { + "commands": { + "bgp": { + "argument": "$", + "lg_query_arg": "argument", + "query": "show ip bgp", + "routeserver": "rs1.dfw.de-cix.net", + "submit": "Submit", + "tab": "prefix_info" + }, + "bgp_v6": { + "argument": "$", + "lg_query_arg": "argument", + "query": "show ip bgp", + "routeserver": "rs1.dfw.de-cix.net -6", + "submit": "Submit", + "tab": "prefix_info" + }, + "neighbor": { + "argument": "$", + "lg_query_arg": "argument", + "query": "show ip bgp neighbor", + "routeserver": "rs1.dfw.de-cix.net", + "submit": "Submit", + "tab": "neighbour_info" + }, + "neighbor_v6": { + "argument": "$", + "lg_query_arg": "argument", + "query": "show ip bgp neighbor", + "routeserver": "rs1.dfw.de-cix.net -6", + "submit": "Submit", + "tab": "neighbour_info" + }, + "summary": { + "lg_query_arg": "argument", + "query": "show ip bgp summary", + "routeserver": "rs1.dfw.de-cix.net", + "submit": "Submit", + "tab": "bgp_summary" + }, + "summary_v6": { + "lg_query_arg": "argument", + "query": "show ip bgp summary", + "routeserver": "rs1.dfw.de-cix.net -6", + "submit": "Submit", + "tab": "bgp_summary" + } + }, + "html": { + "class": "", + "contentindex": 0, + "id": "", + "insertbr": false, + "striphtml": false, + "tag": "pre", + "tagindex": 0 + }, + "http": { + "prefix": true, + "rarg": "routeserver", + "referer": "https://lg.dfw.de-cix.net/", + "type": "post", + "url": "https://lg.dfw.de-cix.net/" + }, + "lat": 50.1109, + "lon": 8.6821, + "output": { + "summary": { + "asn": 1, + "ip": 0, + "ipCount": 5 + }, + "communities": { + "block_all": ["0:62499", "62499:65281", "62499:65282", "65535:65281", "65535:65282"], + "block_one": "0:$AS", + "allow_one": "62499:$AS" + }, "type": "bird" } }, @@ -1830,7 +2100,7 @@ "utf8": "%E2%9C%93" }, "neighbor": { - "authenticity_token": "cNf2v04xuMp8EepN0lr%2FaYbWklwcJbN3fNNUp1lBPw4%3D", + "authenticity_token": "%2FK0QmP9CkrrDqOWF9%2B2T1%2BhdAL%2FhE%2Frrqofgi62lJzY%3D", "form[arg]": "$", "form[query]": "100", "form[router]": "waw-rs1.plix.pl", @@ -1838,7 +2108,7 @@ "utf8": "%E2%9C%93" }, "summary": { - "authenticity_token": "cNf2v04xuMp8EepN0lr%2FaYbWklwcJbN3fNNUp1lBPw4%3D", + "authenticity_token": "%2FK0QmP9CkrrDqOWF9%2B2T1%2BhdAL%2FhE%2Frrqofgi62lJzY%3D", "form[arg]": "", "form[query]": "2", "form[router]": "waw-rs1.plix.pl", @@ -1870,40 +2140,39 @@ "ip": 0, "ipCount": -1 }, + "communities":{ + "allow_one": "$AS:65111", + "block_one": "$AS:65000", + "block_all": ["8545:65000", "8545:65281", "8545:65282", "65535:65281", "65535:65282"] + }, "type": "bird" } }, "8631": { "commands": { "bgp": { - "Submit": "1", - "addr": "$", - "community": "", - "lg_query_arg": "addr", - "query": "bgp", - "rtr": "msk-rs1.ripn.net", - "run.x": "9", - "run.y": "15" + "query":"bgp", + "rtr":"msk-rs1.ripn.net", + "addr":"$", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" }, "neighbor": { - "Submit": "1", - "addr": "$", - "community": "", - "lg_query_arg": "addr", - "query": "ne", - "rtr": "msk-rs1.ripn.net", - "run.x": "15", - "run.y": "12" - }, - "summary": { - "Submit": "1", - "addr": "$", - "community": "", - "lg_query_arg": "addr", - "query": "summary", - "rtr": "msk-rs1.ripn.net", - "run.x": "15", - "run.y": "12" + "query":"ne", + "rtr":"msk-rs1.ripn.net", + "addr":"", + "ne_arg":"$", + "community":"", + "lg_query_arg": "ne_arg" + }, + "summary": { + "query":"summary", + "rtr":"msk-rs1.ripn.net", + "addr":"", + "ne_arg":"", + "community":"", + "lg_query_arg": "addr" } }, "html": { @@ -1912,15 +2181,15 @@ "id": "", "insertbr": false, "striphtml": false, - "tag": "pre", + "tag": "json", "tagindex": 0 }, "http": { "prefix": true, "rarg": "rtr", - "referer": "http://www.msk-ix.ru/eng/lookingglass.html", + "referer": "https://www.msk-ix.ru/en/lookingglass?query=", "type": "post", - "url": "http://www.msk-ix.ru/eng/lookingglass.html" + "url": "https://www.msk-ix.ru/ajax/lookingglass/" }, "lat": 55.80758, "lon": 3746679, @@ -1930,34 +2199,37 @@ "ip": 0, "ipCount": 7 }, + "communities": { + "allow_all":["8631:8631"], + "block_all":["0:8631"], + "allow_one": "8631:$AS", + "block_one": "0:$AS" + }, "type": "bird" } }, "8714": { - "commands": { - "bgp": { - "arg": "$", - "arg2": "", - "lg_query_arg": "arg", - "query": "show ip bgp ", - "query route-server": "query route-server", - "rs": "rs3 - ipv4 & ipv6" - }, - "neighbor": { - "arg": "$", - "arg2": "", - "lg_query_arg": "arg", - "query": "show ip bgp neighbor received-routes", - "query route-server": "query route-server", - "rs": "rs3 - ipv4 & ipv6" - }, - "summary": { - "arg": "$", - "arg2": "", - "lg_query_arg": "arg", - "query": "show ip bgp summary", - "query route-server": "query route-server", - "rs": "rs3 - ipv4 & ipv6" + "commands": { + "neighbor":{ + "rs":"rs3.lon1", + "query":"received", + "arg":"$", + "arg2":"", + "lg_query_arg":"arg" + }, + "bgp":{ + "rs":"rs3.lon1", + "query":"route", + "arg":"$", + "arg2":"", + "lg_query_arg":"arg" + }, + "summary": { + "query":"summary", + "arg":"", + "arg2":"", + "lg_query_arg":0, + "rs": "rs3.lon1" } }, "html": { @@ -1980,11 +2252,16 @@ "lon": 0.1275, "output": { "summary": { - "asn": 9, + "asn": 2, "ip": 0, "ipCount": 9 }, - "type": "bird" + "communities": { + "block_all": ["0:8714", "65535:65281", "65535:65282"], + "block_one": "0:$AS", + "allow_one": "8714:$AS" + }, + "type": "quagga" } }, "9033": { @@ -1992,6 +2269,12 @@ "bgp": { "lg_query_arg": "q", "q": "$" + }, + "neighbor": { + "lg_query_arg": null + }, + "summary":{ + "lg_query_arg": null } }, "html": { @@ -2000,7 +2283,7 @@ "id": "", "insertbr": false, "striphtml": false, - "tag": "pre", + "tag": "json", "tagindex": 0 }, "http": { @@ -2008,7 +2291,7 @@ "rarg": 0, "referer": "http://lg.ecix.net", "type": "get", - "url": "http://lg.ecix.net/prefix_detail/rs1-lg.dus/ipv4" + "url": "https://lg.ecix.net/api/routeservers/0/" }, "lat": 51.2384547, "lon": 6.8143502, @@ -2018,7 +2301,12 @@ "ip": 0, "ipCount": 4 }, - "type": "bird" + "communities": { + "block_all": ["65000:0"], + "block_one": "65000:$AS", + "allow_one": "64960:$AS" + }, + "type": "alice" } }, "9439": { diff --git a/lgParameters.py b/lgParameters.py index 280af53..378d270 100644 --- a/lgParameters.py +++ b/lgParameters.py @@ -22,7 +22,7 @@ # # This software program and documentation are copyrighted by The Regents of # the University of California. The software program and documentation are -# supplied $B!H(Bas is$B!I(B, without any accompanying services from The Regents. The +# supplied "as is", without any accompanying services from The Regents. The # Regents does not warrant that the operation of the program will be # uninterrupted or error-free. The end-user understands that the program # was developed for research purposes and is advised not to rely @@ -35,12 +35,14 @@ # THE POSSIBILITY OF SUCH DAMAGE. THE UNIVERSITY OF CALIFORNIA SPECIFICALLY # DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE -# SOFTWARE PROVIDED HEREUNDER IS ON AN $B!H(BAS IS$B!I(B BASIS, AND THE UNIVERSITY OF +# SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF # CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, # ENHANCEMENTS, OR MODIFICATIONS. -import json, os, sys -from pprint import pprint +import os +import sys +import json + class LgParameters: def getRequestParameters(self, command, jsonstr): @@ -48,7 +50,6 @@ def getRequestParameters(self, command, jsonstr): return queryparams[command] def getLgProfile(self, asn): - #with open(os.path.abspath(os.path.dirname(sys.argv[0]))+"/ixp_parameters.json") as data_file: - with open(os.path.abspath(os.path.dirname(sys.argv[0]))+"/ixp_parameters_new.json") as data_file: + with open(os.path.abspath(os.path.dirname(sys.argv[0])) + "/ixp_parameters_new.json") as data_file: lgData = json.load(data_file) return lgData["looking_glasses"][asn] diff --git a/main.py b/main.py index 851895f..6e93c5e 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,8 @@ -#!/usr/bin/python +#!/usr/bin/env python __author__ = "Vasilis Giotsas" __email__ = "" + # This software is Copyright (C) 2015 The Regents of the University of # California. All Rights Reserved. Permission to copy, modify, and # distribute this software and its documentation for educational, research @@ -24,7 +25,7 @@ # # This software program and documentation are copyrighted by The Regents of # the University of California. The software program and documentation are -# supplied $B!H(Bas is$B!I(B, without any accompanying services from The Regents. The +# supplied "as is", without any accompanying services from The Regents. The # Regents does not warrant that the operation of the program will be # uninterrupted or error-free. The end-user understands that the program # was developed for research purposes and is advised not to rely @@ -37,27 +38,22 @@ # THE POSSIBILITY OF SUCH DAMAGE. THE UNIVERSITY OF CALIFORNIA SPECIFICALLY # DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE -# SOFTWARE PROVIDED HEREUNDER IS ON AN $B!H(BAS IS$B!I(B BASIS, AND THE UNIVERSITY OF +# SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF # CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, # ENHANCEMENTS, OR MODIFICATIONS. -import HttpQueryHandler -from Helper import Helper -from BgpParser import BgpParser -import lgParameters -import datetime -import time -import re -import random -import json import sys +import time +import ujson import getopt -import random -from pprint import pprint -""" -test -""" +import logging +import datetime +from shutil import copyfile from collections import Iterable +import lgParameters +import HttpQueryHandler +from Helper import Helper +from BgpParser import BgpParser def getIptoASN(filename): @@ -71,52 +67,138 @@ def getIptoASN(filename): line = line.strip() if not line.startswith("#"): # skip comments if len(line) > 0: - tokens = line.split(" ") + tokens = line.split() addresses[tokens[0]] = tokens[1] return addresses -def sendQuery(outputfile, asn, lg, command, queryAddress=False): + +def getRegexASNs(filename): + """ + Gets the list of ASNs that will be passed + as argument to the regex looking glass query + """ + as_numbers = set() + with open(filename, "r") as f: + for line in f: + if not line.startswith("#"): + lf = line.strip().split() + if len(lf) > 1: + as_numbers.add(lf[1]) + return as_numbers + + +def set_query_arguments(lg, query_address, command, asn): + """ + Sets the arguments of the LG query based on the + command, the LG type and the ASN. + :param lg: The LG parameters + :type lg: dict + :param query_address: The target of the LG command + :type query_address: string + :param command: The LG command + :type command: string + :param asn: The LG ASN + :type asn: string + :return: The appropriate method to issue the HTTP query, + the initial value of the query argument to reset it, and the + LG url to which the query will be sent. + :rtype: tuple + """ prevq = "" - if command == "traceroute" or command == "neighbor" or lg["http"]["prefix"] == False: - # if IP prefix is given strip out the prefix length - #print queryAddress - if isinstance(queryAddress, Iterable): - if '/' in queryAddress: - queryAddress = Helper.convertPrefixToAddress(address) - # validate the ip address - ipIsValid = Helper.validateIPNetwork(queryAddress) - if ipIsValid or command == "summary" or command == "regex": + if lg["output"]["type"] != "alice": + q_arg = lg["commands"][command]["lg_query_arg"] + q = lg["commands"][command] - if lg["commands"][command]["lg_query_arg"] in q: - prevq = q[lg["commands"][command]["lg_query_arg"]] + if q_arg in q: + prevq = q[q_arg] + if type(lg["http"]["rarg"]) == dict: + lg["http"]["rarg"] = lg["http"]["rarg"][command] if lg["http"]["rarg"] != 0: if not isinstance(q[lg["http"]["rarg"]], basestring): q[lg["http"]["rarg"]] = q[lg["http"]["rarg"]][0] # set the command argument - if command != "summary": - q[lg["commands"][command]["lg_query_arg"]] = q[lg["commands"][command]["lg_query_arg"]].replace("$", queryAddress); - - # send the http request - the_page = qh.send_http_request(q, str(lg["http"]["url"]), str(lg["http"]["referer"]), lg["http"]["type"]) - #print the_page - - q[lg["commands"][command]["lg_query_arg"]] = prevq - # scrape the output - table = qh.parse_html(the_page, lg["html"]) - if lg["html"]["striphtml"] == True: - table = qh.strip_tags(table) - + if not command.startswith("summary"): + q[q_arg] = q[q_arg].replace("$", query_address) + + # send the http request + if asn != "59900": + http_func = "send_http_request" + else: + http_func = "http_request_alt" + + query_url = lg["http"]["url"] + # set the appropriate API endpoint if lg type is 'alice' + if lg["output"]["type"] == "alice": + endpoints = { + "summary": "neighbours", + "neighbor": "neighbours/" + str(query_address) + "/routes" + } + query_url = lg["http"]["url"] + endpoints[command] + return (http_func, prevq, query_url) + + +def scrap_html_oputput(lg, lg_output, asn): + """ + Extracts the command output from the LG response + :param lg: The LG parameters + :type lg: dict + :param lg_output: The body of the LG HTTP response + :param asn: The LG ASN + :type asn: string + :return: The command output of the LG response + :rtype: string + """ + if type(lg["html"]) == list: + for html_rule in lg["html"]: + if "commands" in html_rule: + if command in html_rule["commands"]: + lg["html"] = html_rule + break + else: + raise IOError("Malformed JSON format of IXP configuration file") + command_outout = qh.parse_html(lg_output, lg["html"], asn) + if lg["html"]["striphtml"] is True: + command_outout = qh.strip_tags(command_outout) + if type(command_outout) == dict: + command_outout = ujson.dumps(command_outout) + return command_outout + + +def send_query(outputfile, asn, lg, command, query_address=False): + if command == "traceroute" or command.startswith("neighbor") or lg["http"]["prefix"] is False: + # if IP prefix is given strip out the prefix length + if isinstance(query_address, Iterable): + if '/' in query_address: + query_address = Helper.convertPrefixToAddress(address) + + # validate the ip address + ipIsValid = Helper.validateIPNetwork(query_address) + if ipIsValid or command.startswith("summary") or command == "regex" or (command == "neighbor" and lg["output"]["type"] == "alice"): + http_func, prevq, query_url = set_query_arguments(lg, query_address, command, asn) + q = lg["commands"][command] + query_method = getattr(qh, http_func) + lg_response = query_method(q, str(query_url), str(lg["http"]["referer"]), lg["http"]["type"]) + + # print the_page + q = lg["commands"][command] + if lg["commands"][command]["lg_query_arg"] and len(lg["commands"][command]["lg_query_arg"]) > 0: + q[lg["commands"][command]["lg_query_arg"]] = prevq + + # scrap the output + command_output = scrap_html_oputput(lg, lg_response, asn) + # write to file filename = outputfile filename = filename.replace(":", "-") - if command != "summary": - Helper.saveToFile(filename, "#> " + str(queryAddress) + "\n", "a+", asn) - Helper.saveToFile(filename, str(table), "a+", asn) + if not command.startswith("summary"): + Helper.saveToFile(filename, "#> " + str(query_address) + "\n", "a+", asn) + Helper.saveToFile(filename, str(command_output), "a+", asn) filepath = Helper.saveToFile(filename, "\n-----------------------------------------------------\n\n", "a+", asn) time.sleep(15) return filepath + def usage(): print "usage:" print "main.py -a -c -o [-i ] " @@ -125,15 +207,16 @@ def usage(): print "\t\tbgp\n\t\tsummary\n\t\tneighbor" print "\t-o: path to file where the command's output is logged" print "\t-i: path to file with the addresses to query (an address per line)\n" - print "\t-i: path to file with the addresses to query (an address per line)\n" print "\t-f: path to the second file with the addresses required for the bgp and inference commands\n" + def main(argv): asn, inputfile, outputfile, command, inputfile2 = '', '', '', '', '' found_a, found_i, found_o, found_c, found_i2 = False, False, False, False, False try: - opts, args = getopt.getopt(argv, "ha:c:o:i:f:", ["asn=", "command=", "outputfile=", "inputfile=", "inputfile2="]) + opts, args = getopt.getopt(argv, "ha:c:o:i:f:", [ + "asn=", "command=", "outputfile=", "inputfile=", "inputfile2="]) except getopt.GetoptError: usage() sys.exit(2) @@ -162,10 +245,10 @@ def main(argv): sys.exit(2) elif not found_i: ''' the summary command doesn't require input addresses to query ''' - if command != "summary": + if not command.startswith("summary"): usage() sys.exit(2) - elif not found_i2 and command == "bgp": + elif not found_i2 and command.startswith("bgp"): usage() sys.exit(2) @@ -179,43 +262,60 @@ def main(argv): qh = HttpQueryHandler.HttpQueryHandler() lgpar = lgParameters.LgParameters() now = datetime.datetime.now() - #currentDate = now.strftime("%d-%m-%Y") + # currentDate = now.strftime("%d-%m-%Y") parameters = lgpar.getLgProfile(asn) basename = '.'.join(outputfile.split(".")[:-1]) extension = outputfile.split(".")[-1] - - if command == "summary": - filepath = sendQuery(outputfile, asn, parameters, command) - ip_to_asn = BgpParser.parse_summary(asn, filepath, 4, parameters["output"]) - ipfile = basename+"_addresses."+extension + ip_version = 4 + if command.endswith("_v6"): + ip_version = 6 + bgp_parser = BgpParser() + if command.startswith("summary"): + filepath = send_query(outputfile, asn, parameters, command) + ip_to_asn = BgpParser.parse_summary(filepath, ip_version, parameters["output"]) + ipfile = basename + "_addresses." + extension for ip in ip_to_asn: - #print ip_to_asn[ip] - #Helper.saveToFile(ipfile, ip+" "+ip_to_asn[ip]+"\n", "a+", asn) - Helper.saveToFile(ipfile, ip+" "+str(ip_to_asn[ip])+"\n", "a+", asn) - elif command == "neighbor": + # print ip_to_asn[ip] + # Helper.saveToFile(ipfile, ip+" "+ip_to_asn[ip]+"\n", "a+", asn) + Helper.saveToFile(ipfile, ip + " " + str(ip_to_asn[ip]) + "\n", "a+", asn) + elif command.startswith("neighbor"): # read the IP addresses/prefixes - addresses = dict() addresses = getIptoASN(inputfile) - counter = 1 # just for printing progress + counter = 1 # just for printing progress if len(addresses) < 1: - print "Not enough addresses to query" + logging.warning("No addresses to query") else: + # start_parsing = True for address in addresses: print str(counter) + ". " + asn + " " + ": " + address counter += 1 - filepath = sendQuery(outputfile, asn, parameters, command, address) - elif command == "bgp": + filepath = send_query(outputfile, asn, parameters, command, address) + elif command == "regex": + as_numbers = getRegexASNs(inputfile) + counter = 1 # just for printing progress + if len(as_numbers) < 1: + logging.warning("No ASNs to query") + else: + for member_asn in as_numbers: + print str(counter) + ". " + asn + " " + ": " + member_asn + counter += 1 + filepath = send_query(outputfile, asn, parameters, command, member_asn) + + elif command.startswith("bgp"): addresses = getIptoASN(inputfile2) - neigh_file = basename+"_addresses."+extension - prefixes = BgpParser.parseNeighInfo(inputfile, addresses, asn, neigh_file) - for prefix in prefixes: - filepath = sendQuery(outputfile, asn, parameters, command, prefix) + neigh_file = basename + "_addresses." + extension + if ip_version == 6: + neigh_file = basename + "_addresses-v6." + extension + if parameters["output"]["type"] == "alice": + copyfile(inputfile, "lg-logs/" + asn + "/" + outputfile) + else: + prefixes = BgpParser.parse_neigh_info(inputfile, addresses, asn, neigh_file) + for prefix in prefixes: + filepath = send_query(outputfile, asn, parameters, command, prefix) elif command == "inference": addresses = getIptoASN(inputfile2) - if parameters["output"]["type"] == "quagga": - links = BgpParser.parse_prefix_info_quagga(inputfile, addresses, parameters["output"]["communities"][0], parameters["output"]["communities"][1]) - else: - links = BgpParser.parse_prefix_info(inputfile, addresses, parameters["output"]["communities"][0], parameters["output"]["communities"][1]) + func_name = "parse_prefix_info_%s" % parameters["output"]["type"] + links = getattr(bgp_parser, func_name)(inputfile, addresses, parameters["output"]["communities"]) for link in links: - Helper.saveToFile(outputfile, link+"\n", "a+", asn) + Helper.saveToFile(outputfile, link + "\n", "a+", asn)