From 75f878f6b1d1e53f4167095ed792d41f6bc82a0e Mon Sep 17 00:00:00 2001 From: mpope9 Date: Sat, 18 Dec 2021 16:42:46 -0800 Subject: [PATCH 1/2] Adding proxy support --- stats/args.py | 6 ++++++ stats/general_requester.py | 7 ++----- stats/nba_sql.py | 4 +++- stats/play_by_play.py | 5 ++--- stats/player_game_log.py | 4 +--- stats/player_general_traditional_total.py | 5 ++--- stats/player_season.py | 6 ++---- stats/team.py | 2 +- stats/utils.py | 13 +++++++++++++ 9 files changed, 32 insertions(+), 20 deletions(-) diff --git a/stats/args.py b/stats/args.py index 52ac20b7..8e9262f3 100644 --- a/stats/args.py +++ b/stats/args.py @@ -65,4 +65,10 @@ def create_parser(parser): action='store_true', help='Setting to define stdout logging level. If set, only "ok" will be printed if ran successfully. This currently only applies to refreshing a db, and not loading one.') + parser.add_argument( + '--proxy', + dest='proxy', + default=None, + help='This will set a proxy for every request. This is useful if you want to run this from one of the major cloud providers. stats.nba.com does not allow traffic from them directly. Example value: "http://127.0.0.1:8888" if you are running a local proxy on port 8888.') + return parser diff --git a/stats/general_requester.py b/stats/general_requester.py index 87e4c61a..81b796ee 100644 --- a/stats/general_requester.py +++ b/stats/general_requester.py @@ -1,7 +1,4 @@ -import requests - -from utils import get_rowset_mapping, column_names_from_table -from constants import headers +from utils import get_rowset_mapping, column_names_from_table, get_request from db_utils import insert_many @@ -31,7 +28,7 @@ def generate_rows(self, params): """ # json response - response = requests.get(url=self.url, headers=headers, params=params).json() + response = get_request(self.url, params, self.settings) result_sets = response['resultSets'][0] rowset = result_sets['rowSet'] diff --git a/stats/nba_sql.py b/stats/nba_sql.py index edb4b3ac..239be62e 100644 --- a/stats/nba_sql.py +++ b/stats/nba_sql.py @@ -273,6 +273,7 @@ def main(args): seasons = args.seasons skip_tables = args.skip_tables quiet = args.quiet + proxy = args.proxy if not quiet: print(f"Loading seasons: {seasons}.") @@ -285,7 +286,8 @@ def main(args): args.database_host, args.batch_size, args.sqlite_path, - args.quiet) + args.quiet, + args.proxy) if default_mode_set: default_mode(settings, create_schema, request_gap, seasons, skip_tables) diff --git a/stats/play_by_play.py b/stats/play_by_play.py index 139ee686..11fda09d 100644 --- a/stats/play_by_play.py +++ b/stats/play_by_play.py @@ -1,9 +1,8 @@ -import requests import urllib.parse from models import PlayByPlay -from constants import headers from db_utils import insert_many +from utils import get_request class PlayByPlayRequester: @@ -30,7 +29,7 @@ def fetch_game(self, game_id): # Encode without safe '+', apparently the NBA likes unsafe url params. params_str = urllib.parse.urlencode(params, safe=':+') - response = requests.get(url=self.url, headers=headers, params=params_str).json() + response = get_request(self.url, params, self.settings) # pulling just the data we want player_info = response['resultSets'][0]['rowSet'] diff --git a/stats/player_game_log.py b/stats/player_game_log.py index 0929935d..ba00b027 100644 --- a/stats/player_game_log.py +++ b/stats/player_game_log.py @@ -1,4 +1,3 @@ -import requests import urllib.parse from db_utils import insert_many @@ -6,7 +5,6 @@ from models import PlayerGameLog, PlayerGameLogTemp from game import GameEntry from general_requester import GenericRequester -from constants import headers class PlayerGameLogRequester(GenericRequester): @@ -106,7 +104,7 @@ def fetch_season(self, season_id): # Encode without safe '+', apparently the NBA likes unsafe url params. params_str = urllib.parse.urlencode(params, safe=':+') - response = requests.get(url=self.url, headers=headers, params=params_str).json() + response = get_request(self.url, params, self.settings) result_sets = response['resultSets'][0] rowset = result_sets['rowSet'] diff --git a/stats/player_general_traditional_total.py b/stats/player_general_traditional_total.py index dbb6a652..7a02a72d 100644 --- a/stats/player_general_traditional_total.py +++ b/stats/player_general_traditional_total.py @@ -1,10 +1,9 @@ -import requests import urllib.parse from utils import get_rowset_mapping, column_names_from_table, season_id_to_int from models import PlayerGeneralTraditionalTotal from general_requester import GenericRequester -from constants import headers +from utils import get_request class PlayerGeneralTraditionalTotalRequester(GenericRequester): @@ -31,7 +30,7 @@ def generate_rows(self, season_id): params_str = urllib.parse.urlencode(params, safe=':+') # json response - response = requests.get(url=self.url, headers=headers, params=params_str).json() + response = get_request(self.url, params, self.settings) result_sets = response['resultSets'][0] rowset = result_sets['rowSet'] diff --git a/stats/player_season.py b/stats/player_season.py index 0a44b672..28384877 100644 --- a/stats/player_season.py +++ b/stats/player_season.py @@ -1,10 +1,8 @@ -import requests import urllib.parse -from utils import get_rowset_mapping, column_names_from_table, season_id_to_int +from utils import get_rowset_mapping, column_names_from_table, season_id_to_int, get_request from models import PlayerSeason from general_requester import GenericRequester -from constants import headers class PlayerSeasonRequester(GenericRequester): @@ -31,7 +29,7 @@ def populate_season(self, season_id): params_str = urllib.parse.urlencode(params, safe=':+') # json response - response = requests.get(url=self.url, headers=headers, params=params_str).json() + response = get_request(self.url, params, self.settings) result_sets = response['resultSets'][0] rowset = result_sets['rowSet'] diff --git a/stats/team.py b/stats/team.py index eb5b83f8..10e21d01 100644 --- a/stats/team.py +++ b/stats/team.py @@ -4,7 +4,7 @@ class TeamRequester(GenericRequester): - team_details_url = 'https://stats.nba.com/stats/teamdetails' + team_details_url = 'http://stats.nba.com/stats/teamdetails' def __init__(self, settings): """ diff --git a/stats/utils.py b/stats/utils.py index cfa090ff..cf913a68 100644 --- a/stats/utils.py +++ b/stats/utils.py @@ -2,7 +2,9 @@ Misc utilities. """ +import requests import datetime +from constants import headers def season_id_to_int(season_id): @@ -108,3 +110,14 @@ def generate_valid_season(season): next_year = next_year_rev[::-1] return f"{season}-{next_year}" +def get_request(url, params, settings): + + if settings.proxy: + proxies = { + 'http': settings.proxy, + 'https': settings.proxy, + } + else: + proxies = {} + + return requests.get(url=url, headers=headers, params=params, proxies=proxies).json() From 468b0231fde66979ae0378d3e3b2623290940b74 Mon Sep 17 00:00:00 2001 From: mpope9 Date: Sat, 18 Dec 2021 18:15:06 -0800 Subject: [PATCH 2/2] adding proxy to settings --- stats/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stats/settings.py b/stats/settings.py index 23806cbe..a6a0ad29 100644 --- a/stats/settings.py +++ b/stats/settings.py @@ -16,7 +16,7 @@ class Settings: - def __init__(self, database_type, database_name, database_user, database_password, database_host, batch_size, sqlite_path, quiet): + def __init__(self, database_type, database_name, database_user, database_password, database_host, batch_size, sqlite_path, quiet, proxy): self.user_agent = ( "Mozilla/5.0 (X11; Linux x86_64) " @@ -31,6 +31,7 @@ def __init__(self, database_type, database_name, database_user, database_passwor password = DB_PASSWORD host = DB_HOST self.batch_size = batch_size + self.proxy = proxy if database_name is not None: name = database_name