From 72f45e84c9ac2a96374df8fbf54f7d1285ca5c5c Mon Sep 17 00:00:00 2001 From: Joe-Lonsdale Date: Wed, 3 Jan 2024 17:58:42 +0000 Subject: [PATCH 01/10] remove redundant code --- scripts/battery-mailer/src/run_battery_mailer.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/scripts/battery-mailer/src/run_battery_mailer.py b/scripts/battery-mailer/src/run_battery_mailer.py index 06b35ae8..49d82208 100644 --- a/scripts/battery-mailer/src/run_battery_mailer.py +++ b/scripts/battery-mailer/src/run_battery_mailer.py @@ -31,6 +31,7 @@ HEIGHT = 1080 + def main(): buildings: List[Building] = extract_buildings_from('minimal.kml') @@ -40,7 +41,7 @@ def main(): customer_id = create_new_database_record_for(building, google_chrome) # generate_flyer_for(customer_id) - google_chrome.quit(); + google_chrome.quit() def create_new_database_record_for(building: Building, driver): @@ -159,18 +160,6 @@ def extract_buildings_from(kml_file_path: Path): buildings = [] - for solar_array in solar_arrays: - address = get_address_of(solar_array.location) - formatted_addr = address["formatted_address"] - - if formatted_addr not in buildings_dict: - building_info = { "address": address, "arrays": [] } - buildings_dict[formatted_addr] = building_info - - buildings_dict[formatted_addr]["arrays"].append(solar_array) - - print("Combining arrays into buildings...") - for formatted_addr, building_info in buildings_dict.items(): new_building = Building(formatted_addr, building_info["address"], building_info["arrays"]) buildings.append(new_building) From 363bc887785aa1200e8506f1c9614443cce92e56 Mon Sep 17 00:00:00 2001 From: Joe-Lonsdale Date: Wed, 3 Jan 2024 17:59:06 +0000 Subject: [PATCH 02/10] WIP - write all ai predicted data to DB --- .../src/parse_ai_predictions.py | 27 +++++++++++++++++++ .../src/test_ai_predictions.csv | 20 ++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 scripts/battery-mailer/src/parse_ai_predictions.py create mode 100644 scripts/battery-mailer/src/test_ai_predictions.csv diff --git a/scripts/battery-mailer/src/parse_ai_predictions.py b/scripts/battery-mailer/src/parse_ai_predictions.py new file mode 100644 index 00000000..befde0ee --- /dev/null +++ b/scripts/battery-mailer/src/parse_ai_predictions.py @@ -0,0 +1,27 @@ +import argparse +import csv + +def main(): + if not args.filename.endswith('.csv'): + print('TypeError: File must be a .csv') + return + with open(args.filename, newline='') as f: + reader = csv.reader(f, dialect="unix") + for row in reader: + if row[0] == 'panel_lon_lat': continue + latitude = float(row[0].split(', ')[1].rstrip(')')) + longitude = float(row[0].split(', ')[0].lstrip('(')) + area = float(row[1]) + google_earth_link = row[2] + google_maps_link = row[3] + print(latitude,longitude,area,google_earth_link,google_maps_link) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(prog="python parse_ai_predictions", description="Parses AI predictions of Solar Panels, and optionally adds them to a database.") + parser.add_argument('filename', type=str, help="A file to parse") + parser.add_argument('-db', '--database', action="store_true", default=False, help="Store the parsed predictions into a database. (default: False)") + parser.add_argument('-v', '--verbose', action="store_true") + args = parser.parse_args() + main() \ No newline at end of file diff --git a/scripts/battery-mailer/src/test_ai_predictions.csv b/scripts/battery-mailer/src/test_ai_predictions.csv new file mode 100644 index 00000000..012f2502 --- /dev/null +++ b/scripts/battery-mailer/src/test_ai_predictions.csv @@ -0,0 +1,20 @@ +panel_lon_lat,area_in_m2,google_earth_url,google_map_url +"(-1.6721657923569233, 53.788790662040974)",24.8675259988533,"https://earth.google.com/web/@53.788790662040974,-1.6721657923569233,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.788790662040974,-1.6721657923569233&t=k" +"(-1.6651580712130873, 53.78888912027228)",12.1993479860832,"https://earth.google.com/web/@53.78888912027228,-1.6651580712130873,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.78888912027228,-1.6651580712130873&t=k" +"(-1.6647659136090993, 53.7883311902949)",14.2700139372838,"https://earth.google.com/web/@53.7883311902949,-1.6647659136090993,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.7883311902949,-1.6647659136090993&t=k" +"(-1.6610091863854295, 53.788395566830744)",17.8448428907718,"https://earth.google.com/web/@53.788395566830744,-1.6610091863854295,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.788395566830744,-1.6610091863854295&t=k" +"(-1.672420801946402, 53.789916633646115)",18.333207501904,"https://earth.google.com/web/@53.789916633646115,-1.672420801946402,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.789916633646115,-1.672420801946402&t=k" +"(-1.6725943799022656, 53.78991284679107)",25.6586766688875,"https://earth.google.com/web/@53.78991284679107,-1.6725943799022656,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.78991284679107,-1.6725943799022656&t=k" +"(-1.6617957232255052, 53.79006937013314)",21.1657222464709,"https://earth.google.com/web/@53.79006937013314,-1.6617957232255052,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79006937013314,-1.6617957232255052&t=k" +"(-1.6588490264625322, 53.78984468340016)",25.6098402077742,"https://earth.google.com/web/@53.78984468340016,-1.6588490264625322,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.78984468340016,-1.6588490264625322&t=k" +"(-1.6617142915918903, 53.7912647674142)",20.5113136675537,"https://earth.google.com/web/@53.7912647674142,-1.6617142915918903,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.7912647674142,-1.6617142915918903&t=k" +"(-1.661767865035058, 53.79116883375293)",22.3377973131883,"https://earth.google.com/web/@53.79116883375293,-1.661767865035058,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79116883375293,-1.661767865035058&t=k" +"(-1.6617571503464243, 53.79111455549721)",31.6948632624819,"https://earth.google.com/web/@53.79111455549721,-1.6617571503464243,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79111455549721,-1.6617571503464243&t=k" +"(-1.6617507215332443, 53.79106406409654)",18.938779619708,"https://earth.google.com/web/@53.79106406409654,-1.6617507215332443,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79106406409654,-1.6617507215332443&t=k" +"(-1.6581418570127175, 53.79153868326282)",25.5707710388837,"https://earth.google.com/web/@53.79153868326282,-1.6581418570127175,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79153868326282,-1.6581418570127175&t=k" +"(-1.6683919216314398, 53.79279845707973)",15.3541833739974,"https://earth.google.com/web/@53.79279845707973,-1.6683919216314398,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79279845707973,-1.6683919216314398&t=k" +"(-1.6687262199168067, 53.79268358914321)",11.0565747960337,"https://earth.google.com/web/@53.79268358914321,-1.6687262199168067,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79268358914321,-1.6687262199168067&t=k" +"(-1.6619950164340893, 53.79266213029793)",12.6681780127701,"https://earth.google.com/web/@53.79266213029793,-1.6619950164340893,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79266213029793,-1.6619950164340893&t=k" +"(-1.6711199600519155, 53.79416552512306)",26.6842423522652,"https://earth.google.com/web/@53.79416552512306,-1.6711199600519155,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79416552512306,-1.6711199600519155&t=k" +"(-1.6690476605758133, 53.79412513200253)",12.1700461094152,"https://earth.google.com/web/@53.79412513200253,-1.6690476605758133,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79412513200253,-1.6690476605758133&t=k" +"(-1.660977042319529, 53.793454858658656)",17.3564782796395,"https://earth.google.com/web/@53.793454858658656,-1.660977042319529,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.793454858658656,-1.660977042319529&t=k" From 008a682b731f329630e608cf12d965454008efbd Mon Sep 17 00:00:00 2001 From: Joe-Lonsdale Date: Thu, 4 Jan 2024 11:08:28 +0000 Subject: [PATCH 03/10] finish up parsing the ai predicted panels and put them in the database --- .../src/parse_ai_predictions.py | 108 +++++++++++++++++- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/scripts/battery-mailer/src/parse_ai_predictions.py b/scripts/battery-mailer/src/parse_ai_predictions.py index befde0ee..7817a6ed 100644 --- a/scripts/battery-mailer/src/parse_ai_predictions.py +++ b/scripts/battery-mailer/src/parse_ai_predictions.py @@ -1,10 +1,56 @@ import argparse import csv +import requests +from math_utils import Location, centroid +from enum import Enum +from supabase import create_client, Client +import datetime +class SolarPanel: + def __init__(self, lat, lon, area): + self.lat = lat + self.lon = lon + self.area = area + self.location: Location = Location(lat, lon) + + def __str__(self): + return f'SolarArray(lat: {self.lat}, lon: {self.lon}, area: {self.area})' + +class AutoAuditError(Enum): + AREA_UNDER_6M2 = 4 + AREA_OVER_40M2 = 5 + +class Building: + def __init__(self, address, formatted_addr = None, arrays: [SolarPanel] = None): + self.address = address if formatted_addr else {"postcode": address} + self.formatted_addr = formatted_addr if formatted_addr else address + self.arrays: [SolarPanel] = arrays if arrays else [] + self.location: Location = centroid([array.location for array in self.arrays]) + + def getArea(self) -> float: + return sum([array.area for array in self.arrays]) + +def get_address_of(location: Location): + response = requests.post('http://localhost:3000/solar-proposals/geocoding', json={ + "lat": location.latitude, "lon": location.longitude + }) + response = response.json() + + return response['results'][0] + + +def get_postcode_of(location: Location): + response = requests.get(f'https://api.postcodes.io/postcodes?lon={location.longitude}&lat={location.latitude}') + response = response.json() + if response['result'] == None: + return None + return response['result'][0]['postcode'] def main(): if not args.filename.endswith('.csv'): print('TypeError: File must be a .csv') return + campaign_id = args.filename.split('_')[0] + solar_panels = [] with open(args.filename, newline='') as f: reader = csv.reader(f, dialect="unix") for row in reader: @@ -12,11 +58,65 @@ def main(): latitude = float(row[0].split(', ')[1].rstrip(')')) longitude = float(row[0].split(', ')[0].lstrip('(')) area = float(row[1]) - google_earth_link = row[2] - google_maps_link = row[3] - print(latitude,longitude,area,google_earth_link,google_maps_link) - + solar_panels.append(SolarPanel(latitude, longitude,area)) + + buildings_dict = {} + for panel in solar_panels: + postcode = get_postcode_of(panel.location) + if not postcode: + continue + if postcode not in buildings_dict: + buildings_dict[postcode] = Building(postcode, []) + buildings_dict[postcode].arrays.append(panel) + duplicate_arrays = [] + items_to_remove = [] + for b in buildings_dict: + if len(buildings_dict[b].arrays) > 1: + duplicate_arrays.extend(buildings_dict[b].arrays) + items_to_remove.append(b) + + for x in items_to_remove: + buildings_dict.pop(x) + + for array in duplicate_arrays: + address = get_address_of(array.location) + formatted_addr = address["formatted_address"] + + if formatted_addr not in buildings_dict: + buildings_dict[formatted_addr] = Building(address, formatted_addr) + buildings_dict[formatted_addr].arrays.append(array) + + print(f"Created {len(buildings_dict)} buildings") + for b in buildings_dict: + create_new_database_record_for(buildings_dict[b],campaign_id) + +def catch_auto_audit_errors_on(building) -> AutoAuditError: + errors = [] + if building.getArea() < 6: + errors.append(AutoAuditError.AREA_UNDER_6M2.value) + if building.getArea() > 40: + errors.append(AutoAuditError.AREA_OVER_40M2.value) + return errors + +def create_new_database_record_for(building: Building, campaign_id): + url: str = "http://localhost:54321" + key: str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0" + + supabase: Client = create_client(url, key) + date = str(datetime.datetime.now()).split(' ') + date = f'{date[0]}T{date[1]}Z' + database_entry = { + "campaign_id": campaign_id, + "address_formatted": building.formatted_addr, + "campaign_specific_data": {"solar_array_info": [ str(array) for array in building.arrays ]}, + "current_status": {"name": "AI-PRE-AUDIT", "description": "An AI model predicted this is a building with a solar array on.", "date_started": date}, + "audit_flags": catch_auto_audit_errors_on(building), + "address": building.address + } + print(database_entry) + print() + supabase.table('campaign_customers').insert(database_entry).execute() if __name__ == '__main__': parser = argparse.ArgumentParser(prog="python parse_ai_predictions", description="Parses AI predictions of Solar Panels, and optionally adds them to a database.") From 21fec47b65788d49dd06678d0d7e63074ecd44b6 Mon Sep 17 00:00:00 2001 From: Joe-Lonsdale Date: Thu, 4 Jan 2024 11:22:38 +0000 Subject: [PATCH 04/10] added some progress prints --- .../src/parse_ai_predictions.py | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/scripts/battery-mailer/src/parse_ai_predictions.py b/scripts/battery-mailer/src/parse_ai_predictions.py index 7817a6ed..a2e25783 100644 --- a/scripts/battery-mailer/src/parse_ai_predictions.py +++ b/scripts/battery-mailer/src/parse_ai_predictions.py @@ -53,6 +53,7 @@ def main(): solar_panels = [] with open(args.filename, newline='') as f: reader = csv.reader(f, dialect="unix") + print("Parsing csv...") for row in reader: if row[0] == 'panel_lon_lat': continue latitude = float(row[0].split(', ')[1].rstrip(')')) @@ -61,7 +62,12 @@ def main(): solar_panels.append(SolarPanel(latitude, longitude,area)) buildings_dict = {} + print("Getting postcodes/addresses...") + + i = 0 for panel in solar_panels: + print(f"{i} / {len(solar_panels)}", end="\r") + i+=1 postcode = get_postcode_of(panel.location) if not postcode: continue @@ -77,8 +83,11 @@ def main(): for x in items_to_remove: buildings_dict.pop(x) - + print("Combining duplicates...") + i = 0 for array in duplicate_arrays: + print(f"{i} / {len(duplicate_arrays)}", end="\r") + i+=1 address = get_address_of(array.location) formatted_addr = address["formatted_address"] @@ -87,8 +96,14 @@ def main(): buildings_dict[formatted_addr].arrays.append(array) print(f"Created {len(buildings_dict)} buildings") - for b in buildings_dict: - create_new_database_record_for(buildings_dict[b],campaign_id) + if args.database: + print("Uploading to database...") + i=0 + for b in buildings_dict: + print(f"{i} / {len(buildings_dict)}", end="\r") + i+=1 + create_new_database_record_for(buildings_dict[b],campaign_id) + print("Completed!") def catch_auto_audit_errors_on(building) -> AutoAuditError: errors = [] @@ -114,8 +129,6 @@ def create_new_database_record_for(building: Building, campaign_id): "audit_flags": catch_auto_audit_errors_on(building), "address": building.address } - print(database_entry) - print() supabase.table('campaign_customers').insert(database_entry).execute() if __name__ == '__main__': From e0f106797bbacc3b018c2b67dd9166760cb3a41f Mon Sep 17 00:00:00 2001 From: Joe-Lonsdale Date: Thu, 4 Jan 2024 12:30:17 +0000 Subject: [PATCH 05/10] parallelise some chunks of code --- .../src/parse_ai_predictions.py | 47 +++++++++++++------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/scripts/battery-mailer/src/parse_ai_predictions.py b/scripts/battery-mailer/src/parse_ai_predictions.py index a2e25783..b5858a67 100644 --- a/scripts/battery-mailer/src/parse_ai_predictions.py +++ b/scripts/battery-mailer/src/parse_ai_predictions.py @@ -5,6 +5,7 @@ from enum import Enum from supabase import create_client, Client import datetime +from concurrent.futures import ThreadPoolExecutor class SolarPanel: def __init__(self, lat, lon, area): self.lat = lat @@ -34,6 +35,8 @@ def get_address_of(location: Location): "lat": location.latitude, "lon": location.longitude }) response = response.json() + if not response['results']: + return None return response['results'][0] @@ -49,11 +52,17 @@ def main(): if not args.filename.endswith('.csv'): print('TypeError: File must be a .csv') return + if args.database: + url: str = "http://localhost:54321" + key: str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0" + + supabase: Client = create_client(url, key) campaign_id = args.filename.split('_')[0] solar_panels = [] + with open(args.filename, newline='') as f: reader = csv.reader(f, dialect="unix") - print("Parsing csv...") + print(f"Parsing {args.filename}") for row in reader: if row[0] == 'panel_lon_lat': continue latitude = float(row[0].split(', ')[1].rstrip(')')) @@ -65,10 +74,18 @@ def main(): print("Getting postcodes/addresses...") i = 0 - for panel in solar_panels: - print(f"{i} / {len(solar_panels)}", end="\r") - i+=1 + def parallel_get_postcode(panel): postcode = get_postcode_of(panel.location) + return (postcode, panel) + + def parallel_get_address(panel): + address = get_address_of(panel.location) + return (address,panel) + + with ThreadPoolExecutor() as executor: + results = list(executor.map(parallel_get_postcode, solar_panels)) + + for postcode, panel in results: if not postcode: continue if postcode not in buildings_dict: @@ -85,10 +102,13 @@ def main(): buildings_dict.pop(x) print("Combining duplicates...") i = 0 - for array in duplicate_arrays: - print(f"{i} / {len(duplicate_arrays)}", end="\r") - i+=1 - address = get_address_of(array.location) + + with ThreadPoolExecutor() as executor: + results = list(executor.map(parallel_get_address, duplicate_arrays)) + + for (address,array) in results: + if not address: + continue formatted_addr = address["formatted_address"] if formatted_addr not in buildings_dict: @@ -99,10 +119,12 @@ def main(): if args.database: print("Uploading to database...") i=0 + entries = [] for b in buildings_dict: print(f"{i} / {len(buildings_dict)}", end="\r") i+=1 - create_new_database_record_for(buildings_dict[b],campaign_id) + entries.append(create_new_database_record_for(buildings_dict[b],campaign_id)) + supabase.table('campaign_customers').insert(entries).execute() print("Completed!") def catch_auto_audit_errors_on(building) -> AutoAuditError: @@ -114,11 +136,6 @@ def catch_auto_audit_errors_on(building) -> AutoAuditError: return errors def create_new_database_record_for(building: Building, campaign_id): - url: str = "http://localhost:54321" - key: str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0" - - supabase: Client = create_client(url, key) - date = str(datetime.datetime.now()).split(' ') date = f'{date[0]}T{date[1]}Z' database_entry = { @@ -129,7 +146,7 @@ def create_new_database_record_for(building: Building, campaign_id): "audit_flags": catch_auto_audit_errors_on(building), "address": building.address } - supabase.table('campaign_customers').insert(database_entry).execute() + return database_entry if __name__ == '__main__': parser = argparse.ArgumentParser(prog="python parse_ai_predictions", description="Parses AI predictions of Solar Panels, and optionally adds them to a database.") From 294174f8622117d86a022b78aa01320b0c8080cb Mon Sep 17 00:00:00 2001 From: Joe-Lonsdale Date: Thu, 4 Jan 2024 15:06:22 +0000 Subject: [PATCH 06/10] use postcodes.io bulk reverse postcode lookup, to speedup the process significantly --- .../src/parse_ai_predictions.py | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/scripts/battery-mailer/src/parse_ai_predictions.py b/scripts/battery-mailer/src/parse_ai_predictions.py index b5858a67..53bbda96 100644 --- a/scripts/battery-mailer/src/parse_ai_predictions.py +++ b/scripts/battery-mailer/src/parse_ai_predictions.py @@ -6,6 +6,7 @@ from supabase import create_client, Client import datetime from concurrent.futures import ThreadPoolExecutor +import json class SolarPanel: def __init__(self, lat, lon, area): self.lat = lat @@ -48,6 +49,25 @@ def get_postcode_of(location: Location): return None return response['result'][0]['postcode'] +def bulk_get_postcode_of(locations: [Location]): + if len(locations) > 100: + print("Please provide 1-100 locations") + return None + query = {"geolocations": [{"longitude": x.longitude, "latitude": x.latitude} for x in locations]} + response = requests.post(f'https://api.postcodes.io/postcodes',json=query) + response = response.json() + if response['result'] == None: + return None + return [{'query': x['query'], 'postcode': x['result'][0]['postcode']} for x in response['result'] if x['result'] != None] + +def parallel_get_postcode(panels): + postcodes = bulk_get_postcode_of([panel.location for panel in panels]) + return (postcodes,panels) + +def parallel_get_address(panel): + address = get_address_of(panel.location) + return (address,panel) + def main(): if not args.filename.endswith('.csv'): print('TypeError: File must be a .csv') @@ -73,19 +93,15 @@ def main(): buildings_dict = {} print("Getting postcodes/addresses...") - i = 0 - def parallel_get_postcode(panel): - postcode = get_postcode_of(panel.location) - return (postcode, panel) - - def parallel_get_address(panel): - address = get_address_of(panel.location) - return (address,panel) - + chunk_size = 100 + solar_chunks = [solar_panels[i:i+chunk_size] for i in range(0,len(solar_panels), chunk_size)] with ThreadPoolExecutor() as executor: - results = list(executor.map(parallel_get_postcode, solar_panels)) + results = list(executor.map(parallel_get_postcode, solar_chunks)) + flattened = [item for sublist in results for item in sublist] + postcodes = [item['postcode'] for sublist in flattened for item in sublist if type(item) == dict] + panels = [item for sublist in flattened for item in sublist if type(item) != dict] - for postcode, panel in results: + for panel,postcode in zip(panels,postcodes): if not postcode: continue if postcode not in buildings_dict: From 610099173574b243dab1da4d2ad1957d6b835b1b Mon Sep 17 00:00:00 2001 From: Joe-Lonsdale Date: Thu, 4 Jan 2024 15:53:46 +0000 Subject: [PATCH 07/10] add rate throttling functionality to make sure that google geocoder isn't called more than 50 times per second --- .../src/parse_ai_predictions.py | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/battery-mailer/src/parse_ai_predictions.py b/scripts/battery-mailer/src/parse_ai_predictions.py index 53bbda96..88da0126 100644 --- a/scripts/battery-mailer/src/parse_ai_predictions.py +++ b/scripts/battery-mailer/src/parse_ai_predictions.py @@ -7,6 +7,11 @@ import datetime from concurrent.futures import ThreadPoolExecutor import json +import threading +import time + +last_api_call_time = time.time() +lock = threading.Lock() class SolarPanel: def __init__(self, lat, lon, area): self.lat = lat @@ -32,6 +37,15 @@ def getArea(self) -> float: return sum([array.area for array in self.arrays]) def get_address_of(location: Location): + global last_api_call_time + + with lock: + elapsed = time.time() - last_api_call_time + wait_time = max(0.02 - elapsed, 0) # 0.02 seconds for 50 calls per second + time.sleep(wait_time) + + last_api_call_time = time.time() + response = requests.post('http://localhost:3000/solar-proposals/geocoding', json={ "lat": location.latitude, "lon": location.longitude }) @@ -41,7 +55,6 @@ def get_address_of(location: Location): return response['results'][0] - def get_postcode_of(location: Location): response = requests.get(f'https://api.postcodes.io/postcodes?lon={location.longitude}&lat={location.latitude}') response = response.json() @@ -62,7 +75,7 @@ def bulk_get_postcode_of(locations: [Location]): def parallel_get_postcode(panels): postcodes = bulk_get_postcode_of([panel.location for panel in panels]) - return (postcodes,panels) + return list(zip(postcodes,panels)) def parallel_get_address(panel): address = get_address_of(panel.location) @@ -97,10 +110,11 @@ def main(): solar_chunks = [solar_panels[i:i+chunk_size] for i in range(0,len(solar_panels), chunk_size)] with ThreadPoolExecutor() as executor: results = list(executor.map(parallel_get_postcode, solar_chunks)) - flattened = [item for sublist in results for item in sublist] - postcodes = [item['postcode'] for sublist in flattened for item in sublist if type(item) == dict] - panels = [item for sublist in flattened for item in sublist if type(item) != dict] + panel_postcode_pairs = [pair for sublist in results for pair in sublist] + postcodes, panels = zip(*panel_postcode_pairs) + print(f"Found {len(postcodes)} panels") + postcodes = [p['postcode'] for p in postcodes] for panel,postcode in zip(panels,postcodes): if not postcode: continue @@ -116,7 +130,7 @@ def main(): for x in items_to_remove: buildings_dict.pop(x) - print("Combining duplicates...") + print(f"Combining {len(items_to_remove)} duplicates...") i = 0 with ThreadPoolExecutor() as executor: From 775d1b50aa8b415578570f837d4de500e55aa17e Mon Sep 17 00:00:00 2001 From: Joe-Lonsdale Date: Fri, 5 Jan 2024 09:17:43 +0000 Subject: [PATCH 08/10] remove test csv --- .../src/test_ai_predictions.csv | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 scripts/battery-mailer/src/test_ai_predictions.csv diff --git a/scripts/battery-mailer/src/test_ai_predictions.csv b/scripts/battery-mailer/src/test_ai_predictions.csv deleted file mode 100644 index 012f2502..00000000 --- a/scripts/battery-mailer/src/test_ai_predictions.csv +++ /dev/null @@ -1,20 +0,0 @@ -panel_lon_lat,area_in_m2,google_earth_url,google_map_url -"(-1.6721657923569233, 53.788790662040974)",24.8675259988533,"https://earth.google.com/web/@53.788790662040974,-1.6721657923569233,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.788790662040974,-1.6721657923569233&t=k" -"(-1.6651580712130873, 53.78888912027228)",12.1993479860832,"https://earth.google.com/web/@53.78888912027228,-1.6651580712130873,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.78888912027228,-1.6651580712130873&t=k" -"(-1.6647659136090993, 53.7883311902949)",14.2700139372838,"https://earth.google.com/web/@53.7883311902949,-1.6647659136090993,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.7883311902949,-1.6647659136090993&t=k" -"(-1.6610091863854295, 53.788395566830744)",17.8448428907718,"https://earth.google.com/web/@53.788395566830744,-1.6610091863854295,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.788395566830744,-1.6610091863854295&t=k" -"(-1.672420801946402, 53.789916633646115)",18.333207501904,"https://earth.google.com/web/@53.789916633646115,-1.672420801946402,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.789916633646115,-1.672420801946402&t=k" -"(-1.6725943799022656, 53.78991284679107)",25.6586766688875,"https://earth.google.com/web/@53.78991284679107,-1.6725943799022656,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.78991284679107,-1.6725943799022656&t=k" -"(-1.6617957232255052, 53.79006937013314)",21.1657222464709,"https://earth.google.com/web/@53.79006937013314,-1.6617957232255052,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79006937013314,-1.6617957232255052&t=k" -"(-1.6588490264625322, 53.78984468340016)",25.6098402077742,"https://earth.google.com/web/@53.78984468340016,-1.6588490264625322,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.78984468340016,-1.6588490264625322&t=k" -"(-1.6617142915918903, 53.7912647674142)",20.5113136675537,"https://earth.google.com/web/@53.7912647674142,-1.6617142915918903,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.7912647674142,-1.6617142915918903&t=k" -"(-1.661767865035058, 53.79116883375293)",22.3377973131883,"https://earth.google.com/web/@53.79116883375293,-1.661767865035058,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79116883375293,-1.661767865035058&t=k" -"(-1.6617571503464243, 53.79111455549721)",31.6948632624819,"https://earth.google.com/web/@53.79111455549721,-1.6617571503464243,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79111455549721,-1.6617571503464243&t=k" -"(-1.6617507215332443, 53.79106406409654)",18.938779619708,"https://earth.google.com/web/@53.79106406409654,-1.6617507215332443,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79106406409654,-1.6617507215332443&t=k" -"(-1.6581418570127175, 53.79153868326282)",25.5707710388837,"https://earth.google.com/web/@53.79153868326282,-1.6581418570127175,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79153868326282,-1.6581418570127175&t=k" -"(-1.6683919216314398, 53.79279845707973)",15.3541833739974,"https://earth.google.com/web/@53.79279845707973,-1.6683919216314398,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79279845707973,-1.6683919216314398&t=k" -"(-1.6687262199168067, 53.79268358914321)",11.0565747960337,"https://earth.google.com/web/@53.79268358914321,-1.6687262199168067,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79268358914321,-1.6687262199168067&t=k" -"(-1.6619950164340893, 53.79266213029793)",12.6681780127701,"https://earth.google.com/web/@53.79266213029793,-1.6619950164340893,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79266213029793,-1.6619950164340893&t=k" -"(-1.6711199600519155, 53.79416552512306)",26.6842423522652,"https://earth.google.com/web/@53.79416552512306,-1.6711199600519155,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79416552512306,-1.6711199600519155&t=k" -"(-1.6690476605758133, 53.79412513200253)",12.1700461094152,"https://earth.google.com/web/@53.79412513200253,-1.6690476605758133,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.79412513200253,-1.6690476605758133&t=k" -"(-1.660977042319529, 53.793454858658656)",17.3564782796395,"https://earth.google.com/web/@53.793454858658656,-1.660977042319529,4500d,0.6108652381980153y,0h,0t,0r","https://maps.google.com/?q=53.793454858658656,-1.660977042319529&t=k" From 3c05af76e4dcd895ea258bc11074409ba405fafc Mon Sep 17 00:00:00 2001 From: Joe-Lonsdale Date: Mon, 8 Jan 2024 09:50:25 +0000 Subject: [PATCH 09/10] WIP - geocode properties which don't have addresses --- scripts/battery-mailer/src/parse_ai_predictions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/battery-mailer/src/parse_ai_predictions.py b/scripts/battery-mailer/src/parse_ai_predictions.py index 88da0126..d9471f8d 100644 --- a/scripts/battery-mailer/src/parse_ai_predictions.py +++ b/scripts/battery-mailer/src/parse_ai_predictions.py @@ -12,6 +12,11 @@ last_api_call_time = time.time() lock = threading.Lock() +def setup_threading(): + global last_api_call_time + global lock + last_api_call_time = time.time() + lock = threading.Lock() class SolarPanel: def __init__(self, lat, lon, area): self.lat = lat From 3e90441d55556b5949e8f8e2f7aaf5c4e2df582e Mon Sep 17 00:00:00 2001 From: Joe-Lonsdale Date: Mon, 8 Jan 2024 09:50:39 +0000 Subject: [PATCH 10/10] WIP - geocode properties which don't have addresses --- scripts/battery-mailer/src/geocodeFor.py | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 scripts/battery-mailer/src/geocodeFor.py diff --git a/scripts/battery-mailer/src/geocodeFor.py b/scripts/battery-mailer/src/geocodeFor.py new file mode 100644 index 00000000..e2c6c6ef --- /dev/null +++ b/scripts/battery-mailer/src/geocodeFor.py @@ -0,0 +1,56 @@ +from supabase import create_client +from concurrent.futures import ThreadPoolExecutor +import json +from parse_ai_predictions import setup_threading, Building, parallel_get_address, create_new_database_record_for, SolarPanel + +url = "http://localhost:54321" +key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0" + +supabase = create_client(url,key) + +entries = supabase.table('campaign_customers_postcode_only').select('*').execute() + +entries = entries.data +print(entries[0]) +buildings = [] +for entry in entries: + building = Building(entry['address_formatted'], None, []) + array_info = json.loads(entry['campaign_specific_data'])['solar_array_info'][0] + lat = array_info.split('lat: ')[1].split(',')[0] + lon = array_info.split('lon: ')[1].split(',')[0] + area = array_info.split('area: ')[1].split(')')[0] + building.arrays.append(SolarPanel(lat,lon,area)) + buildings.append(building) +print(buildings[0].arrays) + +setup_threading() + +def main(): + + with ThreadPoolExecutor() as executor: + results = list(executor.map(parallel_get_address, buildings)) + + for (address,array) in results: + if not address: + continue + formatted_addr = address["formatted_address"] + + if formatted_addr not in buildings_dict: + buildings_dict[formatted_addr] = Building(address, formatted_addr) + buildings_dict[formatted_addr].arrays.append(array) + + print(f"Created {len(buildings_dict)} buildings") + if args.database: + print("Uploading to database...") + i=0 + entries = [] + for b in buildings_dict: + print(f"{i} / {len(buildings_dict)}", end="\r") + i+=1 + entries.append(create_new_database_record_for(buildings_dict[b],campaign_id)) + supabase.table('campaign_customers').insert(entries).execute() + print("Completed!") + return + +if __name__ == '__main__': + main() \ No newline at end of file