From c7cdd1c0557bb33279bfe18485d815e8c6a75436 Mon Sep 17 00:00:00 2001 From: northern-64bit <75195383+northern-64bit@users.noreply.github.com> Date: Tue, 24 Aug 2021 20:27:31 +0200 Subject: [PATCH 1/9] Update init.edn --- syncretism-client/resources/init.edn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/syncretism-client/resources/init.edn b/syncretism-client/resources/init.edn index 77f4d70..c63988b 100644 --- a/syncretism-client/resources/init.edn +++ b/syncretism-client/resources/init.edn @@ -5,6 +5,8 @@ :local-crawler false ;; Where do we store permanent data :data-path "data/" + ;; TD Ameritrade API KEY + :td-ameritrade-key "ENTER KEY" ;; All currently loaded data :data From 4f7a47bc2135dadb33d300157204985e653d7406 Mon Sep 17 00:00:00 2001 From: northern-64bit <75195383+northern-64bit@users.noreply.github.com> Date: Wed, 25 Aug 2021 07:20:22 +0200 Subject: [PATCH 2/9] Update init.edn --- syncretism-client/resources/init.edn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/syncretism-client/resources/init.edn b/syncretism-client/resources/init.edn index c63988b..b2039ac 100644 --- a/syncretism-client/resources/init.edn +++ b/syncretism-client/resources/init.edn @@ -7,6 +7,8 @@ :data-path "data/" ;; TD Ameritrade API KEY :td-ameritrade-key "ENTER KEY" + ;; TD Ameritrade API Endpoint + :td-ameritrade-api "https://api.tdameritrade.com/v1/marketdata/" ;; All currently loaded data :data From 4137486734953c09441d40d04379b2127131b79b Mon Sep 17 00:00:00 2001 From: northern-64bit <75195383+northern-64bit@users.noreply.github.com> Date: Wed, 25 Aug 2021 07:20:55 +0200 Subject: [PATCH 3/9] Update communication.clj --- syncretism-client/src/syncretism_client/communication.clj | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/syncretism-client/src/syncretism_client/communication.clj b/syncretism-client/src/syncretism_client/communication.clj index 3ba4cc3..b266396 100644 --- a/syncretism-client/src/syncretism_client/communication.clj +++ b/syncretism-client/src/syncretism_client/communication.clj @@ -39,3 +39,10 @@ http/get :body (json/read-str :key-fn keyword))) + +(defn get-td-ameritrade-ladder + [] + (-> (str :td-ameritrade-api "/chains?apikey=" :td-ameritrade-key "&symbol=" ticker) + http/get + :body + (json/read-str :key-fn keyword))) From 6d0daad917d928ffc34414c96b12e6cd372fe4bb Mon Sep 17 00:00:00 2001 From: northern-64bit <75195383+northern-64bit@users.noreply.github.com> Date: Wed, 25 Aug 2021 10:51:10 +0200 Subject: [PATCH 4/9] Fixed error --- syncretism-client/src/syncretism_client/communication.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncretism-client/src/syncretism_client/communication.clj b/syncretism-client/src/syncretism_client/communication.clj index b266396..d82399c 100644 --- a/syncretism-client/src/syncretism_client/communication.clj +++ b/syncretism-client/src/syncretism_client/communication.clj @@ -42,7 +42,7 @@ (defn get-td-ameritrade-ladder [] - (-> (str :td-ameritrade-api "/chains?apikey=" :td-ameritrade-key "&symbol=" ticker) + (-> (str (:td-ameritrade-api @state/state) "/chains?apikey=" (:td-ameritrade-key @state/state) "&symbol=" ticker) http/get :body (json/read-str :key-fn keyword))) From 496b4357e7281180a528fe5e0e9716a5737aff9c Mon Sep 17 00:00:00 2001 From: northern-64bit <75195383+northern-64bit@users.noreply.github.com> Date: Wed, 25 Aug 2021 15:03:38 +0200 Subject: [PATCH 5/9] Create External APIs.md A description of the TD Ameritrade API --- syncretism-client/External APIs.md | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 syncretism-client/External APIs.md diff --git a/syncretism-client/External APIs.md b/syncretism-client/External APIs.md new file mode 100644 index 0000000..89f1d50 --- /dev/null +++ b/syncretism-client/External APIs.md @@ -0,0 +1,31 @@ +#External API:s + +##TD Ameritrade +### Get Option Ladders +####Example Use: +`curl -X GET https://api.tdameritrade.com/v1/marketdata//chains?apikey=ENTERKEY&symbol=AAPL` + +Returns every option for the ticker `AAPL`. + +###Parameters + +* `apikey [string]`: Enter the API key **Requirement** +* `symbol [string]`: One ticker for the option ladder **Requirement** +* `contractType [string]`: Type of contracts to return in the ladder. Can be CALL, PUT, or ALL. **default:ALL** +* `strikeCount [int]`: The number of strikes to return above and below the at-the-money price +* `includeQuotes [true|false]`: Include quotes for options in the option chain. Can be TRUE or FALSE. **default:FALSE** +* `stratedgy [string]`: Passing a value returns a Strategy Chain. Possible values are SINGLE, ANALYTICAL (allows use of the volatility, underlyingPrice, interestRate, and daysToExpiration params to calculate theoretical values), COVERED, VERTICAL, CALENDAR, STRANGLE, STRADDLE, BUTTERFLY, CONDOR, DIAGONAL, COLLAR, or ROLL. **Default:SINGLE** +* `interval []`: Strike interval for spread strategy chains (see strategy param) +* `strike [int]`: Provide a strike price to return options only at that strike price. +* `range [str]`: Returns options for the given range. Possible values are: +ITM (In-the-money),NTM (Near-the-money), OTM (Out-of-the-money), SAK (Strikes Above Market), SBK (Strikes Below Market), SNK (Strikes Near Market) and ALL (All Strikes) **Default:ALL** +* `fromDate [date]`: Only return expirations after this date. For strategies, expiration refers to the nearest term expiration in the strategy. Valid ISO-8601 formats are: yyyy-MM-dd and yyyy-MM-dd'T'HH:mm:ssz. +* `toDate [date]`: Only return expirations before this date. For strategies, expiration refers to the nearest term expiration in the strategy. Valid ISO-8601 formats are: yyyy-MM-dd and yyyy-MM-dd'T'HH:mm:ssz. +* `volatility [float]`: Volatility to use in calculations. Applies only to ANALYTICAL strategy chains (see strategy param). +* `underlyingPrice [float]`: Underlying price to use in calculations. Applies only to ANALYTICAL strategy chains (see strategy param). +* `interestRate [float]`: Interest rate to use in calculations. Applies only to ANALYTICAL strategy chains (see strategy param). +* `daysToExpiration [float]`: Days to expiration to use in calculations. Applies only to ANALYTICAL strategy chains (see strategy param). +* `expMonth [str]`: Return only options expiring in the specified month. Month is given in the three character format. Example: JAN **Default:ALL** +* `optionType [str]`: Type of contracts to return. Possible values are: S (Standard contracts), NS (Non-standard contracts) and ALL (All contracts) **Default:ALL** + +For more information visit: https://developer.tdameritrade.com \ No newline at end of file From 8861c5cfe819505b77fc8f6bd26548e59f1b0eb1 Mon Sep 17 00:00:00 2001 From: northern-64bit <75195383+northern-64bit@users.noreply.github.com> Date: Mon, 30 Aug 2021 21:16:12 +0200 Subject: [PATCH 6/9] Made Risk Free Rate no longer hardcoded Replaced hardcoded rfr with a function, which returns historical rfr from the API endpoint of St. Louis Fed Data Source FRED. --- syncretism-crawler/resources/config.edn | 4 ++-- syncretism-crawler/src/syncretism/crawler/shared.clj | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/syncretism-crawler/resources/config.edn b/syncretism-crawler/resources/config.edn index 96d9be7..39b414c 100644 --- a/syncretism-crawler/resources/config.edn +++ b/syncretism-crawler/resources/config.edn @@ -19,6 +19,6 @@ ;; Where to save the options data :save-path "options" - ;; Risk free interest rate (ask google) - :risk-free 0.0154 + ;; St. Louis FRED api key + :fred-api-key "ENTER KEY" } diff --git a/syncretism-crawler/src/syncretism/crawler/shared.clj b/syncretism-crawler/src/syncretism/crawler/shared.clj index 4368b19..0c1aed9 100644 --- a/syncretism-crawler/src/syncretism/crawler/shared.clj +++ b/syncretism-crawler/src/syncretism/crawler/shared.clj @@ -5,7 +5,9 @@ [clojure.data.csv :as csv] [taoensso.timbre :as timbre :refer [info warn error]] [taoensso.timbre.appenders.core :as appenders] - [miner.ftp :as ftp])) + [miner.ftp :as ftp] + [clojure.data.json :as json] + [clj-http.client :as http])) (def state (atom @@ -19,6 +21,14 @@ (def config (-> "resources/config.edn" slurp read-string)) +;; rfr returns historical rfr data from St. Louis FRED (for later use) +(defn rfr + [] + (-> (str "https://api.stlouisfed.org/fred/series/observations?series_id=DGS10&api_key=" (:fred-api-key @config) "&file_type=json") + http/get + :body + (json/read-str :key-fn keyword))) + ;; GETTING SYMBOLS LIST ;; -------------------- ;; All date is sourced from http://www.nasdaqtrader.com/trader.aspx?id=symboldirdefs From dfda64d19b5f73111e3a0c39e731cea5c6c813a8 Mon Sep 17 00:00:00 2001 From: northern-64bit <75195383+northern-64bit@users.noreply.github.com> Date: Thu, 2 Sep 2021 08:11:29 +0200 Subject: [PATCH 7/9] Added RFR Cralwer - Added RFR Crawler in crawler.shared.clj - Updated README.md from mentioning dataops-crawler to syncretism-crawler --- README.md | 2 +- .../src/syncretism/crawler/shared.clj | 27 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bc05bf0..af7bfbe 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ We try to minimize dependencies to make it easy to install and run locally. Join us on [Discord](https://discord.gg/qBWD5Sus3d) for any question you might have. Use [Github Issues](https://github.com/Tyruiop/syncretism/issues) for bug reports & feature requests. -## Crawler `/datops-crawler` +## Crawler `/syncretism-crawler` This crawlers targets **Yahoo Finance** market data to extract options related information. diff --git a/syncretism-crawler/src/syncretism/crawler/shared.clj b/syncretism-crawler/src/syncretism/crawler/shared.clj index 0c1aed9..7823417 100644 --- a/syncretism-crawler/src/syncretism/crawler/shared.clj +++ b/syncretism-crawler/src/syncretism/crawler/shared.clj @@ -13,7 +13,8 @@ (atom {;; possible status: `:running`, `:paused`, `:terminate` :fundamentals-status :paused - :options-status :running})) + :options-status :running + :rfr-status :paused})) (timbre/merge-config! {:appenders {:println {:enabled? false} @@ -29,6 +30,30 @@ :body (json/read-str :key-fn keyword))) +(defn rfr-crawler + [] + (case (:rfr-status @state) + :running + (do + (println (format "Gathering the Risk Free Rate")) + (try + (let [data (rfr)] + (json/write-str data)) + (catch Exception e (warn "Error with Risk Free Rate"))) + (Thread/sleep (* 24 60 60 1000)) + (recur)) + + :paused + (do + (Thread/sleep (* 10 1000)) + (recur)) + + :terminate + (do + (info "Terminating Risk Free Rate crawler.") + :done)) + ) + ;; GETTING SYMBOLS LIST ;; -------------------- ;; All date is sourced from http://www.nasdaqtrader.com/trader.aspx?id=symboldirdefs From 5fb3bda81fc975f2de0702fb908e26fba5e4300e Mon Sep 17 00:00:00 2001 From: northern-64bit <75195383+northern-64bit@users.noreply.github.com> Date: Mon, 6 Sep 2021 13:34:25 +0200 Subject: [PATCH 8/9] Fixed Requested Changes --- syncretism-crawler/src/syncretism/crawler/core.clj | 1 + syncretism-crawler/src/syncretism/crawler/shared.clj | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/syncretism-crawler/src/syncretism/crawler/core.clj b/syncretism-crawler/src/syncretism/crawler/core.clj index ef6c01d..808f9c6 100644 --- a/syncretism-crawler/src/syncretism/crawler/core.clj +++ b/syncretism-crawler/src/syncretism/crawler/core.clj @@ -13,6 +13,7 @@ (if (= "--init" (first args)) (db/init-db) (do + (shared/rfr-crawler) (options/init-queue) (future (fundamentals/crawler)) (future (options/crawler))))) diff --git a/syncretism-crawler/src/syncretism/crawler/shared.clj b/syncretism-crawler/src/syncretism/crawler/shared.clj index 7823417..697b320 100644 --- a/syncretism-crawler/src/syncretism/crawler/shared.clj +++ b/syncretism-crawler/src/syncretism/crawler/shared.clj @@ -14,7 +14,7 @@ {;; possible status: `:running`, `:paused`, `:terminate` :fundamentals-status :paused :options-status :running - :rfr-status :paused})) + :rfr-status :running})) (timbre/merge-config! {:appenders {:println {:enabled? false} @@ -32,13 +32,13 @@ (defn rfr-crawler [] + (def risk-free-rate (atom "Not Crawled")) (case (:rfr-status @state) :running (do (println (format "Gathering the Risk Free Rate")) (try - (let [data (rfr)] - (json/write-str data)) + (reset! risk-free-rate (atom rfr)) (catch Exception e (warn "Error with Risk Free Rate"))) (Thread/sleep (* 24 60 60 1000)) (recur)) From 8e9be6446dbd238b86a0ff43aee3d5d333ba4bbb Mon Sep 17 00:00:00 2001 From: northern-64bit <75195383+northern-64bit@users.noreply.github.com> Date: Thu, 9 Sep 2021 20:31:19 +0200 Subject: [PATCH 9/9] Update greeks.clj I think that I fixed it. --- syncretism/src/syncretism/greeks.clj | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/syncretism/src/syncretism/greeks.clj b/syncretism/src/syncretism/greeks.clj index b604814..89343c8 100644 --- a/syncretism/src/syncretism/greeks.clj +++ b/syncretism/src/syncretism/greeks.clj @@ -3,8 +3,6 @@ (:require [fastmath.random :as fr])) -;; Risk free interest rate, -(def rfr 0.0154) (def cdf-normal (fn [x] (fr/cdf (fr/distribution :normal) x))) (defn calc-annual-yield @@ -22,13 +20,13 @@ (defn calc-d1 [{s0 :stock-price q :yield X :strike t :t v :impliedVolatility :as data}] (/ (+ (Math/log (/ s0 X)) - (* t (+ (- rfr q) (/ (Math/pow v 2) 2)))) + (* t (+ (- risk-free-rate q) (/ (Math/pow v 2) 2)))) (* v (Math/sqrt t)))) (defn calc-d2 [d1 v t] (- d1 (* v (Math/sqrt t)))) (defn calc-premium [{opt-type :opt-type X :strike s0 :stock-price eqt :eqt d1 :d1 d2 :d2 t :t q :yield}] - (let [xert (* X (Math/exp (- (* rfr t)))) + (let [xert (* X (Math/exp (- (* risk-free-rate t)))) s0eqt (* s0 eqt)] (cond (= opt-type "C") (- (* s0eqt (cdf-normal d1)) (* xert (cdf-normal d2))) @@ -57,7 +55,7 @@ (let [p1 (- (/ (* s0 v eqt (Math/exp (- (/ (* d1 d1) 2)))) (* 2 (Math/sqrt t) (Math/sqrt (* 2 Math/PI))))) - p2 (* rfr X (Math/exp (- (* rfr t)))) + p2 (* risk-free-rate X (Math/exp (- (* risk-free-rate t)))) p3 (* q s0 eqt)] (* (/ 1 365) @@ -75,7 +73,7 @@ (defn calc-rho [{opt-type :opt-type X :strike t :t d2 :d2}] - (let [xtert (* X t (Math/exp (- (* rfr t))))] + (let [xtert (* X t (Math/exp (- (* risk-free-rate t))))] (cond (= opt-type "C") (/ (* xtert (cdf-normal d2)) 100) @@ -99,6 +97,6 @@ [:rho (calc-rho full-data)] [:premium (calc-premium full-data)] [:dividendYield q] - [:rfr rfr]] + [:rfr risk-free-rate]] (filter #(-> % last Double/isNaN not)) (into {}))))