From e61c47b29e0f736ba39c40377daf7854fd2a0216 Mon Sep 17 00:00:00 2001 From: Eugene Date: Thu, 4 Aug 2016 22:51:24 +0300 Subject: [PATCH 1/2] Added endpoints to download all account history --- oandapy/oandapy.py | 38 ++++++++++++++++++++++++++++++++++++++ oandapy/utils.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 oandapy/utils.py diff --git a/oandapy/oandapy.py b/oandapy/oandapy.py index e75a6af..8a9fd52 100644 --- a/oandapy/oandapy.py +++ b/oandapy/oandapy.py @@ -1,5 +1,8 @@ +import os import json +import zipfile import requests +from .utils import json_to_csv from .exceptions import BadEnvironment, OandaError """ OANDA API wrapper for OANDA's REST API """ @@ -167,6 +170,41 @@ def get_transaction(self, account_id, transaction_id): (account_id, transaction_id) return self.request(endpoint) + def get_account_history_url(self, account_id, **params): + """Returns link to download account history file + Docs: http://developer.oanda.com/rest-live/transaction-history + """ + endpoint = '%s/v1/accounts/%s/alltransactions' % (self.api_url, + account_id) + response = self.client.get(endpoint, params=params) + return response.headers['Location'] + + def get_account_history_file(self, file_url, directory, **params): + """Downloads account history to specified folder in both csv and json + Docs: http://developer.oanda.com/rest-live/transaction-history + """ + params['stream'] = True + response = self.client.get(file_url, params=params) + if response.status_code == 404: + return None + out_file_name = 'acct' + zip_file_name = os.path.join(directory, out_file_name+'.zip') + csv_file = None + with open(zip_file_name, 'a+b') as f: + for chunk in response: + f.write(chunk) + f.seek(0) + zip_file = zipfile.ZipFile(f) + for name in zip_file.namelist(): + zip_file.extract(name, directory) + full_path = os.path.join(directory, name) + new_full_path = os.path.join(directory, out_file_name+'.json') + os.rename(full_path, new_full_path) + csv_file = json_to_csv(new_full_path, out_file_name+'.csv') + zip_file.close() + os.remove(zip_file_name) + return csv_file + """Forex Labs""" def get_eco_calendar(self, **params): diff --git a/oandapy/utils.py b/oandapy/utils.py new file mode 100644 index 0000000..3a87fe5 --- /dev/null +++ b/oandapy/utils.py @@ -0,0 +1,28 @@ +import os +import csv +import json + + +def json_to_csv(json_file_path, csv_name, delete_source=False): + """ + Converts json file to csv. + :param json_file_path: json file which needs to be converted to csv + :param csv_name: name for the csv file + :param delete_source: should json file be deleted or not + :return: path to new csv file + """ + with open(json_file_path, 'r') as f: + data = json.loads(f.read()) + headers = [] + for line in data: + for key in line.iterkeys(): + if key not in headers: + headers.append(key) + csv_file_path = os.path.join(os.path.dirname(json_file_path), csv_name) + with open(csv_file_path, 'w') as f: + csv_file = csv.DictWriter(f, headers) + csv_file.writeheader() + csv_file.writerows(data) + if delete_source: + os.remove(json_file_path) + return csv_file_path From a0462206b7936f2321a26330bd20e8872b3be7e5 Mon Sep 17 00:00:00 2001 From: Eugene Date: Fri, 5 Aug 2016 00:24:19 +0300 Subject: [PATCH 2/2] Added endpoints to download all account history --- oandapy/oandapy.py | 34 +++++++++++++++++++++++++++------- oandapy/utils.py | 8 +++++++- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/oandapy/oandapy.py b/oandapy/oandapy.py index 8a9fd52..8deb508 100644 --- a/oandapy/oandapy.py +++ b/oandapy/oandapy.py @@ -1,8 +1,10 @@ import os +import sys import json +import time import zipfile import requests -from .utils import json_to_csv +from .utils import json_to_csv, ensure_directory_exists from .exceptions import BadEnvironment, OandaError """ OANDA API wrapper for OANDA's REST API """ @@ -170,16 +172,32 @@ def get_transaction(self, account_id, transaction_id): (account_id, transaction_id) return self.request(endpoint) - def get_account_history_url(self, account_id, **params): - """Returns link to download account history file + def get_account_history(self, account_id, directory=None, **params): + """ Returns path to the csv file, containing account history information + If directory is specified, file will be created in that directory, + otherwise it will be created near the main module. Docs: http://developer.oanda.com/rest-live/transaction-history """ endpoint = '%s/v1/accounts/%s/alltransactions' % (self.api_url, account_id) response = self.client.get(endpoint, params=params) - return response.headers['Location'] - - def get_account_history_file(self, file_url, directory, **params): + file_url = response.headers['Location'] + if not directory: + directory = os.path.dirname( + os.path.abspath(sys.modules['__main__'].__file__) + ) + ensure_directory_exists(directory) + max_tries = 5 + timeout = 3 + result = None + for step in xrange(max_tries): + result = self._get_account_history_file(file_url, directory) + if result: + break + time.sleep(timeout) + return result + + def _get_account_history_file(self, file_url, directory, **params): """Downloads account history to specified folder in both csv and json Docs: http://developer.oanda.com/rest-live/transaction-history """ @@ -187,7 +205,7 @@ def get_account_history_file(self, file_url, directory, **params): response = self.client.get(file_url, params=params) if response.status_code == 404: return None - out_file_name = 'acct' + out_file_name = 'account_history' zip_file_name = os.path.join(directory, out_file_name+'.zip') csv_file = None with open(zip_file_name, 'a+b') as f: @@ -199,6 +217,8 @@ def get_account_history_file(self, file_url, directory, **params): zip_file.extract(name, directory) full_path = os.path.join(directory, name) new_full_path = os.path.join(directory, out_file_name+'.json') + if os.path.exists(new_full_path): + os.remove(new_full_path) os.rename(full_path, new_full_path) csv_file = json_to_csv(new_full_path, out_file_name+'.csv') zip_file.close() diff --git a/oandapy/utils.py b/oandapy/utils.py index 3a87fe5..fb4b6e4 100644 --- a/oandapy/utils.py +++ b/oandapy/utils.py @@ -20,9 +20,15 @@ def json_to_csv(json_file_path, csv_name, delete_source=False): headers.append(key) csv_file_path = os.path.join(os.path.dirname(json_file_path), csv_name) with open(csv_file_path, 'w') as f: - csv_file = csv.DictWriter(f, headers) + csv_file = csv.DictWriter(f, headers, + delimiter=',', lineterminator='\n') csv_file.writeheader() csv_file.writerows(data) if delete_source: os.remove(json_file_path) return csv_file_path + + +def ensure_directory_exists(directory): + if not os.path.exists(directory): + os.makedirs(directory)