From f094b01f50a620e37186fc0714dfe32b40849f1b Mon Sep 17 00:00:00 2001 From: huitema Date: Mon, 23 Feb 2026 15:05:59 -0800 Subject: [PATCH 1/2] Add fine grain delay stats --- resolver/rsv_recap.py | 75 ++++++++++++++++++++++++++++++++++++- resolver/rsv_recap_daily.py | 6 +-- 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/resolver/rsv_recap.py b/resolver/rsv_recap.py index cdb830d..ac92860 100644 --- a/resolver/rsv_recap.py +++ b/resolver/rsv_recap.py @@ -95,6 +95,8 @@ def __init__(self, query_cc, query_AS, slice_start, slice_duration, recap_file, self.nb_A_pattern = [ 0, 0, 0, 0, 0, 0, 0 ] # vector of duplicates (ISP, PDNS, Other) self.nb_A_prov = [ 0, 0, 0 ] + # vector of count (ISP, PDNS, Others) by time slices (0-300ms-1s-3s-10s-30s) + self.nb_A_under = [[ 0, 0, 0 ],[ 0, 0, 0 ],[ 0, 0, 0 ],[ 0, 0, 0 ],[ 0, 0, 0 ]] self.zombie = [ 0, 0, 0, 0] self.first_3s = 0 self.first_10s = 0 @@ -128,6 +130,9 @@ def init_next_slice(self): self.previous_slice.nb_A_pattern[i] = self.nb_A_pattern[i] for i in range(0,3): self.previous_slice.nb_A_prov[i] = self.nb_A_prov[i] + for i in range(0,5): + for j in range(0,3): + self.previous_slice.nb_A_under[i][j] += self.nb_A_under[i][j] for i in range(0,4): self.previous_slice.zombie[i] = self.zombie[i] self.previous_slice.first_3s = self.first_3s @@ -150,6 +155,9 @@ def init_next_slice(self): self.nb_A_pattern[i] = 0 for i in range(0,3): self.nb_A_prov[i] = 0 + for i in range(0,5): + for j in range(0,3): + self.nb_A_under[i][j] = 0 self.zombie = [ 0, 0, 0, 0] self.first_3s = 0 self.first_10s = 0 @@ -179,6 +187,7 @@ def summarize(self): self.nb_A_pattern[pattern_id-1] += 1 def update_uid(self, r_uid, query_time, rr_type, resolver_tag): + delta_range = [ 0.3, 1, 3, 10, 30 ] if rr_type == 'HTTPS': r_uid.has_https = True elif rr_type == 'AAAA': @@ -197,6 +206,10 @@ def update_uid(self, r_uid, query_time, rr_type, resolver_tag): r_uid.nb_A_prov[prov_index] += 1 if delta_t <= cutoff_delay: r_uid.has_A_prov[prov_index] = True + for i in range(0, len(delta_range)): + if delta_t <= delta_range[i]: + self.nb_A_under[i][prov_index] += 1 + break def get_header(): s = "CC,AS,start,uids,first_isp," @@ -205,6 +218,11 @@ def get_header(): s += 'first_others,nb_https,nb_AAAA,nb_A,' s += 'A_ISP_only, A_PDNS_only, A_ISP_PDNS, A_others_only, A_ISP_others, A_PDNS_others, A_all3,' s += 'nb_A_ISP, nb_A_PDNS, nb_A_others,' + s += 'nb_A_u300ms_ISP, nb_A_u300ms_PDNS, nb_A_u300ms_others,' + s += 'nb_A_u1s_ISP, nb_A_u1s_PDNS, nb_A_u1s_others,' + s += 'nb_A_u3s_ISP, nb_A_u3s_PDNS, nb_A_u3s_others,' + s += 'nb_A_u10s_ISP, nb_A_u10s_PDNS, nb_A_u10s_others,' + s += 'nb_A_u30s_ISP, nb_A_u30s_PDNS, nb_A_u30s_others,' s += 'zombies,z_ISP,z_PDNS,z_others,first_3s,first_10s,sum_delay,max_delay' + '\n' return s @@ -225,6 +243,9 @@ def save_to_file(self): s += str(pattern_total) + ',' for dups_total in self.nb_A_prov: s += str(dups_total) + ',' + for i in range(0,5): + for prov_index in range(0,3): + s += str(self.nb_A_under[i][prov_index]) + ',' for z in self.zombie: s += str(z) + ',' s += str(self.first_3s) + ',' + str(self.first_10s) + ',' @@ -378,6 +399,11 @@ def save_and_close(self): 'first_others', 'nb_https', 'nb_AAAA', 'nb_A', 'A_ISP_only', ' A_PDNS_only', ' A_ISP_PDNS', ' A_others_only', ' A_ISP_others', ' A_PDNS_others', ' A_all3', 'nb_A_ISP', ' nb_A_PDNS', ' nb_A_others', + 'nb_A_u300ms_ISP', 'nb_A_u300ms_PDNS', 'nb_A_u300ms_others', + 'nb_A_u1s_ISP', 'nb_A_u1s_PDNS', 'nb_A_u1s_others', + 'nb_A_u3s_ISP', 'nb_A_u3s_PDNS', 'nb_A_u3s_others', + 'nb_A_u10s_ISP', 'nb_A_u10s_PDNS', 'nb_A_u10s_others', + 'nb_A_u30s_ISP', 'nb_A_u30s_PDNS', 'nb_A_u30s_others', 'zombies', 'z_ISP', 'z_PDNS', 'z_others', 'first_3s', 'first_10s', 'sum_delay', 'max_delay' ] @@ -388,6 +414,11 @@ def save_and_close(self): recap_final_columns = [ 'nb_https', 'nb_AAAA', 'nb_A', 'A_ISP_only', 'A_PDNS_only', 'A_ISP_PDNS', 'A_others_only', 'A_ISP_others', 'A_PDNS_others', 'A_all3', 'nb_A_ISP', 'nb_A_PDNS', 'nb_A_others', + 'nb_A_u300ms_ISP', 'nb_A_u300ms_PDNS', 'nb_A_u300ms_others', + 'nb_A_u1s_ISP', 'nb_A_u1s_PDNS', 'nb_A_u1s_others', + 'nb_A_u3s_ISP', 'nb_A_u3s_PDNS', 'nb_A_u3s_others', + 'nb_A_u10s_ISP', 'nb_A_u10s_PDNS', 'nb_A_u10s_others', + 'nb_A_u30s_ISP', 'nb_A_u30s_PDNS', 'nb_A_u30s_others', 'zombies', 'z_ISP', 'z_PDNS', 'z_others', 'first_3s', 'first_10s', 'sum_delay' ] @@ -396,6 +427,12 @@ def save_and_close(self): 'googlepdns', 'cloudflare', 'opendns', 'quad9', 'level3', 'neustar', 'he' ] class recap_row: + under_names = [ + ['nb_A_u300ms_ISP', 'nb_A_u300ms_PDNS', 'nb_A_u300ms_others'], + ['nb_A_u1s_ISP', 'nb_A_u1s_PDNS', 'nb_A_u1s_others'], + ['nb_A_u3s_ISP', 'nb_A_u3s_PDNS', 'nb_A_u3s_others'], + ['nb_A_u10s_ISP', 'nb_A_u10s_PDNS', 'nb_A_u10s_others'], + ['nb_A_u30s_ISP', 'nb_A_u30s_PDNS', 'nb_A_u30s_others']] def __init__(self, row): self.query_cc = row['CC'] self.query_AS = row['AS'] @@ -419,6 +456,9 @@ def __init__(self, row): self.nb_A_ISP = row['nb_A_ISP'] self.nb_A_PDNS = row['nb_A_PDNS'] self.nb_A_others = row['nb_A_others'] + for i in range(0,5): + for prov_index in range(0,3): + self.nb_A_under[1][prov_index] = row[recap_row.under_names[i][prov_index]] self.zombies = row['zombies'] self.z_ISP = row['z_ISP'] self.z_PDNS = row['z_PDNS'] @@ -447,6 +487,9 @@ def add_row(self, row): self.nb_A_ISP += row['nb_A_ISP'] self.nb_A_PDNS += row['nb_A_PDNS'] self.nb_A_others += row['nb_A_others'] + for i in range(0,5): + for prov_index in range(0,3): + self.nb_A_under[1][prov_index] += row[recap_row.under_names[i][prov_index]] self.zombies += row['zombies'] self.z_ISP += row['z_ISP'] self.z_PDNS += row['z_PDNS'] @@ -543,7 +586,10 @@ def save_file(self, file_name): s += str(r_row.A_all3) + "," s += str(r_row.nb_A_ISP) + "," s += str(r_row.nb_A_PDNS) + "," - s += str(r_row.nb_A_others) + "," + s += str(r_row.nb_A_others) + "," + for i in range(0,5): + for prov_index in range(0,3): + s += str(self.nb_A_under[1][prov_index]) s += str(r_row.zombies) + "," s += str(r_row.z_ISP) + "," s += str(r_row.z_PDNS) + "," @@ -566,6 +612,11 @@ def summary_columns(): 'first_others', 'nb_https', 'nb_AAAA', 'nb_A', 'A_ISP_only', ' A_PDNS_only', ' A_ISP_PDNS', ' A_others_only', ' A_ISP_others', ' A_PDNS_others', ' A_all3', 'nb_A_ISP', ' nb_A_PDNS', ' nb_A_others', + 'nb_A_u300ms_ISP', 'nb_A_u300ms_PDNS', 'nb_A_u300ms_others', + 'nb_A_u1s_ISP', 'nb_A_u1s_PDNS', 'nb_A_u1s_others', + 'nb_A_u3s_ISP', 'nb_A_u3s_PDNS', 'nb_A_u3s_others', + 'nb_A_u10s_ISP', 'nb_A_u10s_PDNS', 'nb_A_u10s_others', + 'nb_A_u30s_ISP', 'nb_A_u30s_PDNS', 'nb_A_u30s_others', 'zombies', 'z_ISP', 'z_PDNS', 'z_others', 'first_3s', 'first_10s', 'average_delay', 'max_delay' ] return columns @@ -590,6 +641,8 @@ def summary_row(self): nb_A_ISP = 0 nb_A_PDNS = 0 nb_A_others = 0 + nb_A_under = [ + [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0]] zombies = 0 z_ISP = 0 z_PDNS = 0 @@ -623,7 +676,10 @@ def summary_row(self): nb_A_others += r_row.nb_A_others zombies += r_row.zombies z_ISP += r_row.z_ISP - z_PDNS += r_row.z_PDNS + z_PDNS += r_row.z_PDNS + for i in range(0,5): + for j in range(0,3): + nb_A_under[i][j] += r_row.nb_A_under[i][j] z_others += r_row.z_others first_3s += r_row.first_3s first_10s += r_row.first_10s @@ -662,6 +718,21 @@ def summary_row(self): nb_A_ISP, nb_A_PDNS, nb_A_others, + nb_A_under[0][0], + nb_A_under[0][1], + nb_A_under[0][2], + nb_A_under[1][0], + nb_A_under[1][1], + nb_A_under[1][2], + nb_A_under[2][0], + nb_A_under[2][1], + nb_A_under[2][2], + nb_A_under[3][0], + nb_A_under[3][1], + nb_A_under[3][2], + nb_A_under[4][0], + nb_A_under[4][1], + nb_A_under[4][2], zombies, z_ISP, z_PDNS, diff --git a/resolver/rsv_recap_daily.py b/resolver/rsv_recap_daily.py index fde008d..19e7bba 100644 --- a/resolver/rsv_recap_daily.py +++ b/resolver/rsv_recap_daily.py @@ -103,7 +103,7 @@ def load_bucket(bucket): item = item[:-4] if item.startswith("queries"): item = item[7:] - output_file = os.path.join(temp_dir, "recap-daily" + item + ".csv") + output_file = os.path.join(temp_dir, "recap2-daily" + item + ".csv") bucket = file_bucket(ip2a4, ip2a6, as_names, output_file, source_file, bucket_id, time_loaded) bucket_list.append(bucket) bucket_id += 1 @@ -132,7 +132,7 @@ def load_bucket(bucket): nb_files = 0 for key in rcl.cc_as_list: if rcl.cc_as_list[key].total_uids > 10000: - as_file = os.path.join(output_dir, "recap-" + + as_file = os.path.join(output_dir, "recap2-" + rcl.cc_as_list[key].query_cc + "-" + rcl.cc_as_list[key].query_AS + ".csv") rcl.cc_as_list[key].save_file(as_file) @@ -140,7 +140,7 @@ def load_bucket(bucket): nb_files += 1 print("Saved " + str(nb_files) + " CC/AS files.") - summary_file = os.path.join(output_dir, "recap-summary.csv") + summary_file = os.path.join(output_dir, "recap2-summary.csv") df = rcl.summary_df() df.to_csv(summary_file) print("Saved " + str(df.shape[0]) + " CC/AS summaries in " + summary_file) \ No newline at end of file From 486d9196363111bafc3b3bf68dce267ee665b842 Mon Sep 17 00:00:00 2001 From: huitema Date: Mon, 23 Feb 2026 15:22:08 -0800 Subject: [PATCH 2/2] Remove TLD GOO WOLTERSKLUWER --- lib/DnsStats.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/DnsStats.cpp b/lib/DnsStats.cpp index ab984a1..b3d504e 100644 --- a/lib/DnsStats.cpp +++ b/lib/DnsStats.cpp @@ -237,7 +237,7 @@ char const * RegisteredTldName[] = { "GAP", "GARDEN", "GAY", "GB", "GBIZ", "GD", "GDN", "GE", "GEA", "GENT", "GENTING", "GEORGE", "GF", "GG", "GGEE", "GH", "GI", "GIFT", "GIFTS", "GIVES", "GIVING", "GL", "GLASS", "GLE", "GLOBAL", "GLOBO", "GM", "GMAIL", "GMBH", "GMO", "GMX", "GN", "GODADDY", - "GOLD", "GOLDPOINT", "GOLF", "GOO", "GOODYEAR", "GOOG", "GOOGLE", "GOP", + "GOLD", "GOLDPOINT", "GOLF", "GOODYEAR", "GOOG", "GOOGLE", "GOP", "GOT", "GOV", "GP", "GQ", "GR", "GRAINGER", "GRAPHICS", "GRATIS", "GREEN", "GRIPE", "GROCERY", "GROUP", "GS", "GT", "GU", "GUCCI", "GUGE", "GUIDE", "GUITARS", "GURU", "GW", "GY", "HAIR", "HAMBURG", "HANGOUT", "HAUS", "HBO", "HDFC", "HDFCBANK", @@ -327,7 +327,7 @@ char const * RegisteredTldName[] = { "VOTING", "VOTO", "VOYAGE", "VU", "WALES", "WALMART", "WALTER", "WANG", "WANGGOU", "WATCH", "WATCHES", "WEATHER", "WEATHERCHANNEL", "WEBCAM", "WEBER", "WEBSITE", "WED", "WEDDING", "WEIBO", "WEIR", "WF", "WHOSWHO", "WIEN", "WIKI", - "WILLIAMHILL", "WIN", "WINDOWS", "WINE", "WINNERS", "WME", "WOLTERSKLUWER", "WOODSIDE", + "WILLIAMHILL", "WIN", "WINDOWS", "WINE", "WINNERS", "WME", "WOODSIDE", "WORK", "WORKS", "WORLD", "WOW", "WS", "WTC", "WTF", "XBOX", "XEROX", "XIHUAN", "XIN", "XN--11B4C3D", "XN--1CK2E1B", "XN--1QQW23A", "XN--2SCRJ9C", "XN--30RR7Y", "XN--3BST00M", "XN--3DS443G", "XN--3E0B707E", "XN--3HCRJ9C",