Merge production changes into master

Pipfile

@@ -14,10 +14,10 @@ certifi = "==2020.4.5.2"
 cffi = "==1.14.0"
 chardet = "==3.0.4"
 click = "==7.1.2"
-cryptography = "==2.9.2"
+cryptography = ">=3.2.1"
 cssutils = "==1.0.2"
 diff-match-patch = "==20181111"
-httplib2 = "==0.18.1"
+httplib2 = ">=0.18.1"
 idna = "==2.9"
 itsdangerous = "==0.24"
 kombu = "==4.6.10"
@@ -33,32 +33,38 @@ python-dateutil = "==2.8.1"
 pytz = "==2017.2"
 qrcode = "==6.1"
 delta = {git = "https://github.com/forgeworks/quill-delta-python.git"}
-requests = "==2.23.0"
+requests = ">=2.23.0"
 retrying = "==1.3.3"
 six = "==1.15.0"
 twilio = "==5.7.0"
 urllib3 = "==1.25.9"
 vine = "==1.3.0"
 Cycler = "==0.10.0"
 Cython = "==0.29.20"
-Flask = "==0.12.2"
+Flask = "==1.1.2"
+flask-caching = "==1.9.0"
 Flask-Login = "==0.4.1"
 Flask-Mail = "==0.9.1"
 Flask-QRcode = "==3.0.0"
 Flask-SQLAlchemy = "==2.3.2"
-Flask-WTF = "==0.14.2"
-Jinja2 = "==2.9.6"
+Flask-WTF = "==0.14.3"
+Jinja2 = ">=2.10.1"
 MarkupSafe = "==1.1.1"
 Pillow = "==7.1.2"
 PyJWT = "==1.7.1"
 PySocks = "==1.7.1"
-SQLAlchemy = "==1.3.17"
+SQLAlchemy = ">=1.3.0"
 SQLAlchemy-Utils = "==0.36.5"
-Werkzeug = "==0.12.2"
-WTForms = "==2.3.1"
+Werkzeug = ">=1.0.1"
+WTForms = ">=2.3.1"
 simplekv = "*"
-wtforms-sqlalchemy = "*"
-wtforms-alchemy = "*"
+scipy = "*"
+WTForms-SQLAlchemy = "*"
+WTForms-Alchemy = "*"
+pyOpenSSL = ">=19.1.0"
+importlib-metadata = "==2.0.0"
+flask-migrate = "*"
+email-validator = "*"
 
 [requires]
 python_version = "3.6"

assets/main.css

@@ -1,6 +1,7 @@
 @font-face{
   font-family: Neue_Helvetica;
   src:url("/assets/fonts/neue_helvetica_reg.woff2");
+  font-display: swap;
 }
 body{
   font-family: Neue_Helvetica;
@@ -64,7 +65,7 @@ select {
 .img-circle {
 	horizontal-align:middle;
 	border-radius:50%;
-	height:150px;
+	height:100px;
 }
 
 .col-lg-4 {
@@ -414,4 +415,4 @@ form tr {
   border-style:none none none solid; 
   margin-left:120px; 
   padding:20px;
-}
+}

cleanup_sr.py

@@ -0,0 +1,32 @@
+from medtracker import *
+from sqlalchemy import func, desc
+import pandas as pd
+
+dbobj = models.SurveyResponse.query.all()
+
+objects = pd.DataFrame([i.to_dict() for i in dbobj])
+delete_count = 0
+to_delete = set()
+for name,group in objects.groupby("session_id"):
+	if len(group)>1:
+		group = group.sort_values("start_time",ascending=False)
+		print(group)
+		successful = group[(group["completed"]==1)|(group["exited"]==1)]
+		if len(successful)>0:
+			keep_id = group[(group["completed"]==1)|(group["exited"]==1)].iloc[0,].name
+			print("keeping ID:", keep_id)
+		else:
+			keep_id = None
+		for ix,row in group.iterrows():
+			if row.name==keep_id: continue
+			print("deleting ID: ",row.name)
+			to_delete.add(row.name)
+			delete_count += 1
+	else:
+		row = group.iloc[0,]
+		if (row["completed"]==False) & (row["exited"]==False):
+			print("deleting ID: ",row.name)
+			to_delete.add(row.name)
+			delete_count += 1
+
+delete_objs = [dbobj[i] for i in to_delete]

disable.py

@@ -0,0 +1,28 @@
+from medtracker import *
+from medtracker import database
+patients = models.Patient.query.all()
+devices = models.Device.query.all()
+
+kept = 0
+discarded = 0
+patients = models.Patient.query.all()
+for p in patients:
+    if p.deactivate==True:
+        continue
+    sr = [s.to_dict() for s in p.surveys]
+    df = pd.DataFrame(sr)
+    if len(df)>0:
+        if any(df.start_time > datetime.date(2020,8,1)):
+            kept += 1
+            p.deactivate = False
+            db.session.add(p)
+        else:
+            discarded += 1
+            p.deactivate = True
+            db.session.add(p)
+    else:
+        discarded += 1
+        p.deactivate = True
+        db.session.add(p)
+db.session.commit()
+print(kept,discarded)

medtracker/__init__.py

@@ -1,5 +1,5 @@
 from flask import *
-from flask.ext.cache import Cache
+from flask_caching import Cache
 from requests.auth import HTTPBasicAuth
 import random, string, pytz, sys, random, urllib.parse, datetime, os
 from werkzeug.utils import secure_filename
@@ -8,6 +8,8 @@
 from flask_login import login_user, logout_user, current_user
 from medtracker.config import *
 from flask_qrcode import QRcode
+from flask_migrate import Migrate
+from flask_sqlalchemy import SQLAlchemy
 
 import twilio.twiml
 from twilio.rest import TwilioRestClient
@@ -18,6 +20,9 @@
 from ftplib import FTP_TLS
 from flask import flash
 from itsdangerous import URLSafeTimedSerializer
+
+#from flask_debugtoolbar import DebugToolbarExtension
+
 ts = URLSafeTimedSerializer(flask_secret_key)
 
 #Flask init
@@ -32,7 +37,20 @@
 app.config['SESSION_COOKIE_SECURE'] = False
 app.config['SECRET_KEY'] = flask_secret_key
 app.config['WTF_CSRF_ENABLED']=True
-app.debug = False
+app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 3600
+#app.config["DEBUG_TB_PROFILER_ENABLED"] = True
+#app.config["DEBUG_TB_INTERCEPT_REDIRECTS"] = False
+#app.debug = True
+#toolbar = DebugToolbarExtension(app)
+
+app.debug=False
+
+
+#init db
+db = SQLAlchemy(app)
+db_session = db.session
+
+migrate = Migrate(app,db,render_as_batch=True)
 
 app.config.update(
 	#EMAIL SETTINGS
@@ -54,7 +72,6 @@
 client = TwilioRestClient(twilio_AccountSID, twilio_AuthToken)
 auth_combo=(twilio_AccountSID, twilio_AuthToken)
 
-from medtracker.database import db_session 	# to make sqlalchemy DB calls
 from medtracker.views import *				# web pages
 from medtracker.triggers import *
 

medtracker/data/nyc_plots.py

@@ -0,0 +1,180 @@
+import pandas as pd
+import plotly.graph_objects as go
+import numpy as np
+import requests
+from io import StringIO
+
+def prepFigure(figure, title):
+    figure.update_xaxes(tickformat='%a<br>%b %d',
+        tick0 = '2020-03-22',
+        dtick = 7 * 24 * 3600000,
+        rangeselector=dict(
+            buttons=list([
+                dict(count=7, label="1w", step="day", stepmode="backward"),
+                dict(count=14, label="2w", step="day", stepmode="backward"),
+                dict(count=1, label="1m", step="month", stepmode="backward"),
+                dict(count=1, label="YTD", step="year", stepmode="todate"),
+                dict(step="all")
+            ])
+        )
+    )
+    figure.update_layout(title="<b>" + title + "</b>", hovermode="x",  
+                         legend_orientation="h",
+                         margin={"r":10,"l":30})
+    return figure
+
+start_date = "2020-03-01"
+
+#confirmed cases
+read_headers = {
+    "sec-ch-ua": "\"Chromium\";v=\"92\", \" Not A;Brand\";v=\"99\", \"Google Chrome\";v=\"92\"",
+    "sec-ch-ua-mobile": "?0",
+    "upgrade-insecure-requests": "1"
+  }
+url = "https://static.usafacts.org/public/data/covid-19/covid_confirmed_usafacts.csv"
+req = requests.get(url, headers=read_headers)
+data = StringIO(req.text)
+confirmed_df=pd.read_csv(data)
+
+nyc_counties_df = confirmed_df.loc[(confirmed_df['State'] == 'NY') & (confirmed_df['countyFIPS'].isin([36005, 36061, 36081, 36085, 36047]))]
+nyc_counties_df["County Name"] = [a.strip() for a in nyc_counties_df["County Name"]]
+nyc_counties_df=nyc_counties_df.set_index('County Name')
+nyc_counties_df=nyc_counties_df.drop(columns=['countyFIPS', 'State', 'StateFIPS'])
+nyc_counties_df = nyc_counties_df.transpose()
+nyc_counties_df = nyc_counties_df.rename_axis('Date')
+nyc_counties_df.index =pd.to_datetime(nyc_counties_df.index)
+#print(nyc_counties_df)
+nyc_counties_df['Total'] = nyc_counties_df['Bronx County'] + nyc_counties_df['Kings County'] + nyc_counties_df['New York County'] + nyc_counties_df['Queens County'] + nyc_counties_df['Richmond County']
+
+
+#confirmed deaths
+url = "https://static.usafacts.org/public/data/covid-19/covid_deaths_usafacts.csv"
+req = requests.get(url, headers=read_headers)
+data = StringIO(req.text)
+deaths_df = pd.read_csv(data)
+
+nyc_counties_deaths_df = deaths_df.loc[(deaths_df['State'] == 'NY') & (deaths_df['countyFIPS'].isin([36005, 36061, 36081, 36085, 36047]))]
+nyc_counties_deaths_df["County Name"] = [a.strip() for a in nyc_counties_deaths_df["County Name"]]
+nyc_counties_deaths_df=nyc_counties_deaths_df.set_index('County Name')
+nyc_counties_deaths_df=nyc_counties_deaths_df.drop(columns=['countyFIPS', 'State', 'StateFIPS'])
+
+nyc_counties_deaths_df = nyc_counties_deaths_df.transpose()
+nyc_counties_deaths_df = nyc_counties_deaths_df.rename_axis('Date')
+nyc_counties_deaths_df.index =  pd.to_datetime(nyc_counties_deaths_df.index)
+
+nyc_counties_deaths_df['Total'] = nyc_counties_deaths_df['Bronx County'] + nyc_counties_deaths_df['Kings County'] + nyc_counties_deaths_df['New York County'] + nyc_counties_deaths_df['Queens County'] + nyc_counties_deaths_df['Richmond County']
+
+#case fatality ratio
+cfr_df = nyc_counties_deaths_df / nyc_counties_df 
+
+#daily cases
+def dailydata(dfcounty):
+    dfcountydaily=dfcounty.diff(axis=0)#.fillna(0)
+    return dfcountydaily
+
+DailyCases_df=dailydata(nyc_counties_df)
+DailyDeaths_df=dailydata(nyc_counties_deaths_df)
+DailyCases_df = DailyCases_df.loc[start_date:]
+# compute 7-day exponential moving average
+DailyCases_df['EMA'] = DailyCases_df['Total'].ewm(span=7).mean()
+
+#daily deaths
+DailyDeaths_df = DailyDeaths_df.loc[start_date:]
+DailyDeaths_df.replace(0.0,np.nan, inplace=True)
+# compute 7-day exponential moving average
+DailyDeaths_df['EMA'] = DailyDeaths_df['Total'].ewm(span=7).mean()
+DailyDeaths_df = DailyDeaths_df.loc[start_date:]
+DailyDeaths_df.replace(0.0,np.nan, inplace=True)
+# compute 7-day exponential moving average
+DailyDeaths_df['EMA'] = DailyDeaths_df['Total'].ewm(span=7).mean()
+
+#NYC case hospitalizations
+url = "https://raw.githubusercontent.com/nychealth/coronavirus-data/master/trends/data-by-day.csv"
+
+nyc_case_hosp_death_df = pd.read_csv(url)
+nyc_case_hosp_death_df.index =pd.to_datetime(nyc_case_hosp_death_df.iloc[:,0])
+nyc_case_hosp_death_df = nyc_case_hosp_death_df.drop(nyc_case_hosp_death_df.columns[0], axis=1)
+nyc_case_hosp_death_df = nyc_case_hosp_death_df.rename_axis('Date')
+
+#r0 estimation
+url = "https://d14wlfuexuxgcm.cloudfront.net/covid/rt.csv"
+rt_df = pd.read_csv(url)
+rt_df = rt_df.rename(columns={"mean":"R0_mean"})
+
+#### PLOTS ####
+# case fatality ratio
+df2 = cfr_df.loc["2020-03-14":,]
+fig2 = go.Figure()
+fig2.add_scatter(x=df2.index, y=df2['Bronx County'], mode='lines', name='Bronx')
+fig2.add_scatter(x=df2.index, y=df2['Kings County'], mode='lines',name='Brooklyn')
+fig2.add_scatter(x=df2.index, y=df2['Queens County'], mode='lines', name='Queens')
+fig2.add_scatter(x=df2.index, y=df2['New York County'], mode='lines', name='Manhattan')
+fig2.add_scatter(x=df2.index, y=df2['Richmond County'], mode='lines', name='Staten Island')
+fig2.add_scatter(x=df2.index, y=df2['Total'], mode='lines+markers', name='NYC')
+prepFigure(fig2, title='NYC Case Fatality Rate - From USA Facts')
+fig2.update_layout(yaxis_title="Percent", yaxis_tickformat=".2%")
+fig2.write_json("/home/ubuntu/medtracker/medtracker/data/cfr.json")
+
+#daily new cases
+df2 = DailyCases_df
+fig2 = go.Figure()
+fig2.add_scatter(x=df2.index,y=df2['Bronx County'], mode='lines', name='Bronx')
+fig2.add_scatter(x=df2.index,y=df2['Kings County'], mode='lines', name='Brooklyn')
+fig2.add_scatter(x=df2.index,y=df2['New York County'], mode='lines', name='Manhattan')
+fig2.add_scatter(x=df2.index,y=df2['Queens County'], mode='lines', name='Queens')
+fig2.add_scatter(x=df2.index,y=df2['Richmond County'], mode='lines', name='Staten Island')
+fig2.add_scatter(x=df2.index,y=df2['Total'], mode='lines+markers', name='NYC')
+fig2.add_scatter(x=df2.index, y=df2['EMA'], mode='lines', line=dict(color='royalblue', width=4, dash='dot'), name='7 day EMA')
+prepFigure(fig2, title="NYC Daily New Cases (Source: USA Facts)")
+latest = df2.iloc[-1,:]
+fig2.add_annotation(text="On "+latest.name.strftime("%m/%d")+":\n"+str(int(latest.Total)),
+                   font=dict(family="Helvetica, Arial",size=28),
+                  xref="paper", yref="paper",
+                  x=1, y=1, showarrow=False)
+fig2.write_json("/home/ubuntu/medtracker/medtracker/data/daily_cases.json")
+
+#daily new deaths
+df2 = DailyDeaths_df
+#df2 = temp_df
+fig2 = go.Figure()
+fig2.add_scatter(x=df2.index,y=df2['Bronx County'], mode='lines', name='Bronx')
+fig2.add_scatter(x=df2.index,y=df2['Kings County'], mode='lines', name='Brooklyn')
+fig2.add_scatter(x=df2.index,y=df2['New York County'], mode='lines', name='Manhattan')
+fig2.add_scatter(x=df2.index,y=df2['Queens County'], mode='lines', name='Queens')
+fig2.add_scatter(x=df2.index,y=df2['Richmond County'], mode='lines', name='Staten Island')
+fig2.add_scatter(x=df2.index,y=df2['Total'], mode='lines+markers', name='NYC')
+fig2.add_scatter(x=df2.index, y=df2['EMA'], mode='lines', line=dict(color='royalblue', width=4, dash='dot'), name='7 day EMA')
+prepFigure(fig2, title="NYC Daily Deaths (Source: USA Facts)")
+fig2.write_json("/home/ubuntu/medtracker/medtracker/data/daily_deaths.json")
+
+#hospitalizations
+df2 = nyc_case_hosp_death_df
+fig2 = go.Figure()
+fig2.add_scatter(x=df2.index,y=df2['HOSPITALIZED_COUNT'], mode='lines', name='Hospitalizations', )
+prepFigure(fig2,title="NYC New Hospitalizations (Source: NYC DOHMH)")
+fig2.write_json("/home/ubuntu/medtracker/medtracker/data/hospitalizations.json")
+
+#r0 estimation
+df2 = rt_df[rt_df["region"]=="NY"].sort_values("date").set_index("date")
+fig2 = go.Figure()
+fig2.add_scattergl(x=df2.index,y=df2.R0_mean.where(df2.R0_mean <= 1), line={'color': 'black'}) # below threshold
+fig2.add_scattergl(x=df2.index,y=df2.R0_mean.where(df2.R0_mean >= 1), line={'color': 'red'}) # Above threshhgold
+fig2.add_shape( #dashed line
+        type='line',
+        x0=str(df2.index.min()),
+        y0=1,
+        x1=str(df2.index.max()),
+        y1=1,
+        line=dict(
+            color='black',
+            dash="dashdot"
+        )
+)
+prepFigure(fig2,title="R0 estimate (Source: rt.live)")
+fig2.update_layout(showlegend=False)
+ro_latest = str(round(list(df2["R0_mean"])[-1],2))
+fig2.add_annotation(text="Current R0:\n"+ro_latest,
+                   font=dict(family="Helvetica, Arial",size=24),
+                  xref="paper", yref="paper",
+                  x=1, y=1, showarrow=False)
+fig2.write_json("/home/ubuntu/medtracker/medtracker/data/r0_estimate.json")

medtracker/database.py

@@ -1,8 +1 @@
-from medtracker import *
-from flask_sqlalchemy import SQLAlchemy
-from medtracker.config import *
-
-db = SQLAlchemy(app)
-
-# import a db_session to query db from other modules
-db_session = db.session
+#empty