diff --git a/README.md b/README.md index 411a16e..304f908 100644 --- a/README.md +++ b/README.md @@ -1,66 +1,21 @@ -# dataquery-api +# DataQuery API Client -A simple Python API client for the [JPMorgan DataQuery](https://www.jpmorgan.com/solutions/cib/markets/dataquery)[API](https://developer.jpmorgan.com/products/dataquery_api). +This repository contains example programs in various languages that show how to make calls to the JPMorgan DataQuery API. -# Getting Started +## Running the examples -## Requirements: +Each subdirectory contains a README.md file with instructions specific to each language on how to run the example. -1. OAuth Credentials (Client ID and Client Secret) from [JPMorgan Developer Portal](https://developer.jpmorgan.com/) with access to the DataQuery API. +## API Documentation -1. Python 3.6+ +The API documentation is available at [JPMorgan Developer/DataQuery API](https://developer.jpmorgan.com/products/dataquery_api) -1. An active internet connection +## Issues -## Setting up: +These examples are provided as-is and are not meant to handle all possible error conditions. The [Macrosynergy Package](https://github.com/macrosynergy/macrosynergy) contains a more robust implementation. -1. Clone the repository +If you find any issues with the examples, please report them in the [Issues](https://github.com/macrosynergy/dataquery-api/issues) section of this repository. -```bash -git clone https://github.com/macrosynergy/dataquery-api.git -``` +## License -2. Install the dependencies - -```bash -python -m pip install -r requirements.txt -``` - -## Running `example.py`: - -You'll need to replace the `client_id` and `client_secret` in `dataquery_api.py` with your own OAuth credentials. This can be using a config.yml/json file, environment variables, or hardcoding them in the file (not recommended). - -```python -# example.py -from dataquery_api import DQInterface -import pandas as pd - -client_id = "" # replace with your client id & secret -client_secret = "" - -# initialize the DQInterface object with your client id & secret -dq: DQInterface = DQInterface(client_id, client_secret) - -# check that you have a valid token and can access the API -assert dq.heartbeat(), "DataQuery API Heartbeat failed." - -# create a list of expressions -expressions = [ - "DB(JPMAQS,USD_EQXR_VT10,value)", - "DB(JPMAQS,AUD_EXALLOPENNESS_NSA_1YMA,value)", -] - - -# dates as strings in the format YYYY-MM-DD -start_date: str = "2020-01-25" -end_date: str = "2023-02-05" - -# download the data -data: pd.DataFrame = dq.download( - expressions=expressions, start_date=start_date, end_date=end_date -) - - -print(data.head()) - -``` +The code in this repository is licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for the full license text. \ No newline at end of file diff --git a/go/dataquery_api.go b/go/dataquery_api.go new file mode 100644 index 0000000..f39dd3c --- /dev/null +++ b/go/dataquery_api.go @@ -0,0 +1,232 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "time" +) + +// Constants. WARNING : DO NOT MODIFY. +const ( + oauthBaseURL = "https://api-developer.jpmorgan.com/research/dataquery-authe/api/v2" + timeSeriesEndpoint = "/expressions/time-series" + heartbeatEndpoint = "/services/heartbeat" + oauthTokenURL = "https://authe.jpmchase.com/as/token.oauth2" + oauthDqResourceID = "JPMC:URI:RS-06785-DataQueryExternalApi-PROD" + apiDelayParam = 0.3 // 300ms delay between requests. + exprLimit = 20 // Maximum number of expressions per request (not per "download"). +) + +// DQInterface is a wrapper around the JPMorgan DataQuery API. +type DQInterface struct { + clientID string + clientSecret string + // proxy string + dqResourceID string + currentToken *Token + tokenData url.Values +} + +// Token represents the access token returned by the JPMorgan DataQuery API. +type Token struct { + AccessToken string `json:"access_token"` + CreatedAt time.Time `json:"created_at"` + ExpiresIn int `json:"expires_in"` +} + +// RequestWrapper is a wrapper function around the http.NewRequest() function. +func requestWrapper(url string, headers http.Header, params url.Values, method string) (*http.Response, error) { + req, err := http.NewRequest(method, url, nil) + if err != nil { + return nil, err + } + req.Header = headers + req.URL.RawQuery = params.Encode() + + client := &http.Client{} + res, err := client.Do(req) + if err != nil { + return nil, err + } + + if res.StatusCode != 200 { + return nil, errors.New(fmt.Sprintf("Request failed with status code %v.", res.StatusCode)) + } + + return res, nil +} + +// NewDQInterface returns a new instance of the DQInterface type. +func NewDQInterface(clientID string, clientSecret string) *DQInterface { + return &DQInterface{ + clientID: clientID, + clientSecret: clientSecret, + currentToken: nil, + tokenData: url.Values{ + "grant_type": {"client_credentials"}, + "client_id": {clientID}, + "client_secret": {clientSecret}, + "aud": {oauthDqResourceID}, + }, + } +} + +// GetAccessToken returns an access token for the JPMorgan DataQuery API. +func (dq *DQInterface) GetAccessToken() string { + if dq.currentToken != nil && dq.currentToken.isActive() { + return dq.currentToken.AccessToken + } else { + res, err := requestWrapper(oauthTokenURL, http.Header{}, + dq.tokenData, "POST") + if err != nil { + panic(err) + } + defer res.Body.Close() + + var token Token + json.NewDecoder(res.Body).Decode(&token) + + token.CreatedAt = time.Now() + dq.currentToken = &token + + return dq.currentToken.AccessToken + } +} + +// IsActive returns true if the access token is active. +func (t *Token) isActive() bool { + if t == nil { + return false + } else { + return time.Since(t.CreatedAt).Seconds() < float64(t.ExpiresIn) + } +} + +// Heartbeat returns true if the heartbeat request is successful. +func (dq *DQInterface) Heartbeat() bool { + res, err := requestWrapper(oauthBaseURL+heartbeatEndpoint, + http.Header{"Authorization": {"Bearer " + dq.GetAccessToken()}}, + url.Values{"data": {"NO_REFERENCE_DATA"}}, "GET") + if err != nil { + panic(err) + } + defer res.Body.Close() + + var response map[string]interface{} + json.NewDecoder(res.Body).Decode(&response) + + return response["info"] != nil +} + +// Download returns a list of dictionaries containing the data for the given expressions. + +func (dq *DQInterface) Download(expressions []string, format string, + start_date string, end_date string) []map[string]interface{} { + + calender := "CAL_ALLDAYS" + frequency := "FREQ_DAY" + conversion := "CONV_LASTBUS_ABS" + nan_treatment := "NA_NOTHING" + ref_data := "NO_REFERENCE_DATA" + params := url.Values{ + "format": {format}, + "start-date": {start_date}, + "end-date": {end_date}, + "calendar": {calender}, + "frequency": {frequency}, + "conversion": {conversion}, + "nan_treatment": {nan_treatment}, + "data": {ref_data}} + + if !dq.Heartbeat() { + panic("Heartbeat failed.") + } + + var data []map[string]interface{} + + for i := 0; i < len(expressions); i += exprLimit { + end := i + exprLimit + if end > len(expressions) { + end = len(expressions) + } + + exprSlice := expressions[i:end] + // sleep to avoid hitting the rate limit. + + time.Sleep(time.Duration(apiDelayParam * 1000 * 1000 * 1000)) + + // bool get_pagination = true + var get_pagination bool = true + for get_pagination { + var curr_params url.Values = params + curr_params["expressions"] = exprSlice + var curr_url string = oauthBaseURL + timeSeriesEndpoint + res, err := requestWrapper(curr_url, + http.Header{"Authorization": {"Bearer " + dq.GetAccessToken()}}, + curr_params, "GET") + if err != nil { + panic(err) + } + defer res.Body.Close() + var response map[string]interface{} + json.NewDecoder(res.Body).Decode(&response) + v, ok := response["instruments"] + if response == nil || !ok { + panic("Invalid response from DataQuery API.") + } else { + data = append(data, v.([]map[string]interface{})...) + + if v, ok := response["links"]; ok { + if v.([]interface{})[1].(map[string]interface{})["next"] != nil { + curr_url = v.([]interface{})[1].(map[string]interface{})["next"].(string) + } else { + get_pagination = false + } + } else { + get_pagination = false + } + } + } + } + return data +} + +// func (dq *DQInterface) extract_timeseries(downloaded_data []map[string]interface{}) []map[string]interface{} { +// // create a list of lists to store the timeseries data +// var timeseries_data []map[string]interface{} +// for i := 0; i < len(downloaded_data); i++ { +// // get dictionary for the current instrument +// d := downloaded_data[i].(map[[]]interface{}) +// // get the expression for the current instrument +// expr := d["attributes"][0]["expression"] +// for j := 0; j < len(d["attributes"][0]["timeseries"].([]interface{})); j++ { +// // the first item in the list is the date. convert it from dateteime64[ns] to string +// date := d["attributes"][0]["timeseries"].([]interface{})[j].([]interface{})[0].(time.Time).Format("2006-01-02") +// // the second item in the list is the value. read it as a float64 +// value := d["attributes"][0]["timeseries"].([]interface{})[j].([]interface{})[1].(float64) +// // create a dictionary to store the data +// var timeseries_dict map[string]interface{} +// timeseries_dict["date"] = date +// timeseries_dict["value"] = value +// timeseries_dict["expression"] = expr +// // append the dictionary to the list +// timeseries_data = append(timeseries_data, timeseries_dict) +// } +// } +// return timeseries_data +// } + +// create a main function to test the code +func main() { + + client_id := "" + client_secret := "" + + // create a DQInterface object + dq := NewDQInterface(client_id, client_secret) + dq.Heartbeat() + +} diff --git a/javascript/dataquery_api.js b/javascript/dataquery_api.js new file mode 100644 index 0000000..d76d693 --- /dev/null +++ b/javascript/dataquery_api.js @@ -0,0 +1,238 @@ +const OAUTH_BASE_URL = "https://api-developer.jpmorgan.com/research/dataquery-authe/api/v2"; +const TIMESERIES_ENDPOINT = "/expressions/time-series"; +const HEARTBEAT_ENDPOINT = "/services/heartbeat"; +const OAUTH_TOKEN_URL = "https://authe.jpmchase.com/as/token.oauth2"; +const OAUTH_DQ_RESOURCE_ID = "JPMC:URI:RS-06785-DataQueryExternalApi-PROD"; +const API_DELAY_PARAM = 0.3; +const EXPR_LIMIT = 20; + +function requestWrapper(url, headers = null, params = null, method = "GET") { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open(method, url); + xhr.setRequestHeader("Content-Type", "application/json"); + xhr.setRequestHeader("Accept", "application/json"); + xhr.setRequestHeader("Authorization", `Bearer ${headers.Authorization}`); + + xhr.onload = function () { + if (xhr.status === 200) { + resolve(xhr.response); + } else { + reject(xhr.response); + } + }; + xhr.onerror = function () { + reject("Request failed."); + }; + + if (params) { + xhr.send(JSON.stringify(params)); + } else { + xhr.send(); + } + }); +} + +class DQInterface { + constructor(client_id, client_secret, proxy = null, dq_resource_id = OAUTH_DQ_RESOURCE_ID) { + this.client_id = client_id; + this.client_secret = client_secret; + this.proxy = proxy; + this.dq_resource_id = dq_resource_id; + this.current_token = null; + this.token_data = { + grant_type: "client_credentials", + client_id: this.client_id, + client_secret: this.client_secret, + aud: this.dq_resource_id, + }; + } + + getAccessToken() { + const isActive = (token) => { + if (!token) { + return false; + } else { + const created = new Date(token.created_at); + const expires = token.expires_in; + return (new Date() - created) / 60000 >= expires - 1; + } + }; + + if (isActive(this.current_token)) { + return this.current_token.access_token; + } else { + return new Promise((resolve, reject) => { + requestWrapper(OAUTH_TOKEN_URL, null, this.token_data, "POST") + .then((response) => { + const r_json = JSON.parse(response); + this.current_token = { + access_token: r_json.access_token, + created_at: new Date(), + expires_in: r_json.expires_in, + }; + resolve(this.current_token.access_token); + }) + .catch((error) => { + reject(error); + }); + }); + } + } + + request(url, params, method, headers = null) { + return new Promise((resolve, reject) => { + this.getAccessToken() + .then((access_token) => { + const full_url = OAUTH_BASE_URL + url; + requestWrapper(full_url, { Authorization: access_token }, params, method) + .then((response) => { + resolve(JSON.parse(response)); + }) + .catch((error) => { + reject(error); + }); + }) + .catch((error) => { + reject(error); + }); + }); + } + + heartbeat() { + // use the request function to make a heartbeat request + response = new Promise((resolve, reject) => { + this.request(HEARTBEAT_ENDPOINT, null, "GET") + .then((response) => { + resolve(response); + }) + .catch((error) => { + reject(error); + }); + }); + // if the "info" in response dict, return true else return false + return response["info"] == "OK"; + } + + + + download( + expressions, + start_date, + end_date, + calender = "CAL_ALLDAYS", + frequency = "FREQ_DAY", + conversion = "CONV_LASTBUS_ABS", + nan_treatment = "NA_NOTHING", + ) { + + // declare a dictionary to store predefined parameters + let params_dict = { + "format": "JSON", + "start-date": start_date, + "end-date": end_date, + "calendar": calender, + "frequency": frequency, + "conversion": conversion, + "nan_treatment": nan_treatment, + "data": "NO_REFERENCE_DATA", + } + + // create a list of lists to store the expressions of batches = expr_limit. + let expr_list = [] + let expr_batch = [] + for (let i = 0; i < expressions.length; i++) { + expr_batch.push(expressions[i]) + if (expr_batch.length == EXPR_LIMIT) { + expr_list.push(expr_batch) + expr_batch = [] + } + } + if (expr_batch.length > 0) { + expr_list.push(expr_batch) + } + + // assert that heartbeat is working + if (!this.heartbeat()) { + throw new Error("Heartbeat failed.") + } + + // create a list to store the downloaded data + let downloaded_data = [] + + // loop through the batches of expressions + for (let i = 0; i < expr_list.length; i++) { + // create a copy of the params_dict + let current_params = Object.assign({}, params_dict); + // add the expressions to the copy of params_dict + current_params["expressions"] = expr_list[i]; + // create a url + let curr_url = OAUTH_BASE_URL + TIMESERIES_ENDPOINT; + // create a list to store the current response + let curr_response = {}; + // loop to get next page from the response if any + let get_pagination = true; + while (get_pagination) { + // sleep(API_DELAY_PARAM) + curr_response = this.request(curr_url, current_params, "GET"); + if (curr_response === null || !("instruments" in curr_response)) { + throw new Error("Invalid response."); + } else { + downloaded_data = downloaded_data.concat(curr_response["instruments"]); + if ("links" in curr_response) { + if (curr_response["links"][1]["next"] === null) { + get_pagination = false; + break; + } else { + curr_url = OAUTH_BASE_URL + curr_response["links"][1]["next"]; + current_params = {}; + } + } + } + } + } + return downloaded_data; + } + + to_array(downloaded_data) { + /* for d in dict list + d["attributes"][0]["time-series"] has 2 values - first is datetime64[ns] and second is value of the expression + + create an output list of expression, date and value + */ + let output = [] + for (let i = 0; i < downloaded_data.length; i++) { + let d = downloaded_data[i]; + let expr = d["attributes"][0]["expression"]; + let date = d["attributes"][0]["time-series"][0]; + let value = d["attributes"][0]["time-series"][1]; + output.push([expr, date, value]); + } + return output; + } +} + +// create a main function to run the code +async function main() { + let client_id = ""; + let client_secret = ""; + + // create an instance of the class + let dqClient = new DQInterface(client_id, client_secret); + + // check heartbeat + let heartbeat = await dqClient.heartbeat(); + console.log(heartbeat); + + // download data + let expressions = [ "DB(JPMAQS,USD_EQXR_VT10,value)", + "DB(JPMAQS,AUD_EXALLOPENNESS_NSA_1YMA,value)"]; + let start_date = "2020-01-01"; + let end_date = "2020-12-31"; + let downloaded_data = await dqClient.download(expressions, start_date, end_date); + + // convert the downloaded data to an array + let output = dqClient.to_array(downloaded_data); + + console.log(output.slice(0, 10)); +} \ No newline at end of file diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..411a16e --- /dev/null +++ b/python/README.md @@ -0,0 +1,66 @@ +# dataquery-api + +A simple Python API client for the [JPMorgan DataQuery](https://www.jpmorgan.com/solutions/cib/markets/dataquery)[API](https://developer.jpmorgan.com/products/dataquery_api). + +# Getting Started + +## Requirements: + +1. OAuth Credentials (Client ID and Client Secret) from [JPMorgan Developer Portal](https://developer.jpmorgan.com/) with access to the DataQuery API. + +1. Python 3.6+ + +1. An active internet connection + +## Setting up: + +1. Clone the repository + +```bash +git clone https://github.com/macrosynergy/dataquery-api.git +``` + +2. Install the dependencies + +```bash +python -m pip install -r requirements.txt +``` + +## Running `example.py`: + +You'll need to replace the `client_id` and `client_secret` in `dataquery_api.py` with your own OAuth credentials. This can be using a config.yml/json file, environment variables, or hardcoding them in the file (not recommended). + +```python +# example.py +from dataquery_api import DQInterface +import pandas as pd + +client_id = "" # replace with your client id & secret +client_secret = "" + +# initialize the DQInterface object with your client id & secret +dq: DQInterface = DQInterface(client_id, client_secret) + +# check that you have a valid token and can access the API +assert dq.heartbeat(), "DataQuery API Heartbeat failed." + +# create a list of expressions +expressions = [ + "DB(JPMAQS,USD_EQXR_VT10,value)", + "DB(JPMAQS,AUD_EXALLOPENNESS_NSA_1YMA,value)", +] + + +# dates as strings in the format YYYY-MM-DD +start_date: str = "2020-01-25" +end_date: str = "2023-02-05" + +# download the data +data: pd.DataFrame = dq.download( + expressions=expressions, start_date=start_date, end_date=end_date +) + + +print(data.head()) + +``` diff --git a/dataquery_api.py b/python/dataquery_api.py similarity index 100% rename from dataquery_api.py rename to python/dataquery_api.py diff --git a/example.py b/python/example.py similarity index 100% rename from example.py rename to python/example.py diff --git a/requirements.txt b/python/requirements.txt similarity index 100% rename from requirements.txt rename to python/requirements.txt