Add rvdss indicator/data source

.github/workflows/ci.yaml

@@ -65,7 +65,7 @@ jobs:
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
           docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis
-          
+
 
       - run: |
           wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for

integrations/acquisition/covid_hosp/facility/test_scenarios.py

@@ -42,7 +42,7 @@ def setUp(self):
         cur.execute('truncate table covid_hosp_facility_key')
         cur.execute('truncate table covid_hosp_meta')
         cur.execute('delete from api_user')
-        cur.execute('insert into api_user(api_key, email) values ("key", "emai")')
+        cur.execute('insert into api_user(api_key, email) values ("key", "email")')
 
   @freeze_time("2021-03-16")
   def test_acquire_dataset(self):

integrations/acquisition/rvdss/__init__.py

@@ -0,0 +1,4 @@
+import sys
+import os
+
+sys.path.append(os.getcwd())

integrations/acquisition/rvdss/test_scenarios.py

@@ -0,0 +1,221 @@
+"""Integration tests for acquisition of rvdss data."""
+# standard library
+import unittest
+from unittest.mock import MagicMock, patch
+from copy import copy
+
+# first party
+from delphi.epidata.client.delphi_epidata import Epidata
+from delphi.epidata.acquisition.rvdss.database import update, rvdss_cols, get_num_rows
+import delphi.operations.secrets as secrets
+from delphi_utils import get_structured_logger
+
+# third party
+import mysql.connector 
+from mysql.connector.errors import IntegrityError
+import pandas as pd
+import numpy as np
+from pathlib import Path
+import pdb
+
+# py3tester coverage target (equivalent to `import *`)
+# __test_target__ = 'delphi.epidata.acquisition.covid_hosp.facility.update'
+
+NEWLINE="\n"
+
+class AcquisitionTests(unittest.TestCase):
+  logger = get_structured_logger()
+
+  def setUp(self):
+    """Perform per-test setup."""
+
+    # configure test data
+    # self.test_utils = UnitTestUtils(__file__)
+
+    # use the local instance of the Epidata API
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
+    Epidata.auth = ('epidata', 'key')
+
+    # use the local instance of the epidata database
+    secrets.db.host = 'delphi_database_epidata'
+    secrets.db.epi = ('user', 'pass')
+
+    # clear relevant tables
+    epidata_cnx = mysql.connector.connect(
+        user='user',
+        password='pass',
+        host='delphi_database_epidata',
+        database='epidata')
+    epidata_cur = epidata_cnx.cursor()
+
+    epidata_cur.execute('truncate table rvdss')
+    epidata_cur.execute('DELETE from api_user')
+    epidata_cur.execute('INSERT INTO api_user(api_key, email) VALUES ("key", "email")')
+    epidata_cnx.commit()
+    epidata_cur.close()
+    #epidata_cnx.close()
+
+    # make connection and cursor available to test cases
+    self.cnx = epidata_cnx
+    self.cur = epidata_cnx.cursor()
+
+  def tearDown(self):
+    """Perform per-test teardown."""
+    self.cur.close()
+    self.cnx.close()
+
+  @patch("mysql.connector.connect")
+  def test_rvdss_repiratory_detections(self, mock_sql):
+    connection_mock = MagicMock()
+
+    TEST_DIR = Path(__file__).parent.parent.parent.parent
+    detection_data = pd.read_csv(str(TEST_DIR) + "/testdata/acquisition/rvdss/RVD_CurrentWeekTable_Formatted.csv")
+    detection_data['time_type'] = "week"
+
+    # get the index of the subset of data we want to use
+    subset_index = detection_data[(detection_data['geo_value'].isin(['nl', 'nb'])) & 
+                   (detection_data['time_value'].isin([20240831, 20240907]))].index
+
+
+    # change issue so the data has more than one
+    detection_data.loc[subset_index,"issue"] = 20250227
+
+    # take a small subset just for testing insertion
+    detection_subset = detection_data.loc[subset_index]
+
+    # get the expected response when calling the API
+    # the dataframe needs to add the missing columns and replace nan with None
+    # since that is what is returned from the API
+    df = detection_subset.reindex(rvdss_cols,axis=1)
+    df = df.replace({np.nan: None}).sort_values(by=["epiweek","geo_value"])
+    df = df.to_dict(orient = "records")
+
+    expected_response = {"epidata": df,
+        "result": 1,
+        "message": "success",
+    }
+
+    # get another subset of the data not in the subset to test more calling options
+    detection_subset2 = detection_data[(detection_data['geo_value'].isin(['nu', 'nt'])) & (detection_data['time_value'].isin([20240831, 20240907])) ]
+
+    df2 = detection_subset2.reindex(rvdss_cols,axis=1)
+    df2 = df2.replace({np.nan: None}).sort_values(by=["epiweek","geo_value"])
+    df2 = df2.to_dict(orient = "records")
+
+    expected_response2 = {"epidata": df2,
+        "result": 1,
+        "message": "success",
+    }
+
+    # get another subset of the data for a single geo_value with multiple issues
+    subset_index2 = detection_data[(detection_data['geo_value'].isin(['ouest du québec'])) & 
+                   (detection_data['time_value'].isin([20240831, 20240907]))].index
+
+    detection_data.loc[subset_index2,"issue"] = [20250220,20250227]
+    detection_data.loc[subset_index2,"epiweek"] = [202435,202435]
+    detection_data.loc[subset_index2,"time_value"] = [20240831,20240831]
+
+    detection_subset3 = detection_data.loc[subset_index2]
+    df3 = detection_subset3.reindex(rvdss_cols,axis=1)
+    df3 = df3.replace({np.nan: None}).sort_values(by=["epiweek","geo_value"])
+    df3 = df3.to_dict(orient = "records")
+
+    expected_response3 = {"epidata": df3,
+        "result": 1,
+        "message": "success",
+    }
+
+    # make sure the data does not yet exist
+    with self.subTest(name='no data yet'):
+      response = Epidata.rvdss(geo_type='province',
+                               time_values= [202435, 202436],
+                               geo_value = ['nl','nb'])
+      self.assertEqual(response['result'], -2, response)
+
+    # acquire sample data into local database
+    with self.subTest(name='first acquisition'):
+        # When the MagicMock connection's `cursor()` method is called, return
+        # a real cursor made from the current open connection `cnx`.
+        connection_mock.cursor.return_value = self.cnx.cursor()
+        # Commit via the current open connection `cnx`, from which the cursor
+        # is derived
+        connection_mock.commit = self.cnx.commit
+        mock_sql.return_value = connection_mock
+
+        update(detection_subset, self.logger)
+
+        response = Epidata.rvdss(geo_type='province',
+                                 time_values= [202435, 202436],
+                                 geo_value = ['nl','nb'])
+
+        self.assertEqual(response,expected_response)
+
+    with self.subTest(name='duplicate aquisition'):
+        # The main run function checks if the update has already been fetched/updated
+        # so it should never run twice, and duplocate aquisitions should never 
+        # occur. Running the update twice will result in an error
+
+        # When the MagicMock connection's `cursor()` method is called, return
+        # a real cursor made from the current open connection `cnx`.
+        connection_mock.cursor.return_value = self.cnx.cursor()
+        # Commit via the current open connection `cnx`, from which the cursor
+        # is derived
+        connection_mock.commit = self.cnx.commit
+        mock_sql.return_value = connection_mock
+
+        with self.assertRaises(mysql.connector.errors.IntegrityError):
+            update(detection_subset, self.logger)
+
+    # Request with exact column order
+    with self.subTest(name='exact column order'):
+        rvdss_cols_subset = [col for col in detection_subset2.columns if col in rvdss_cols]
+        ordered_cols = [col for col in rvdss_cols if col in rvdss_cols_subset] 
+        ordered_df = detection_subset2[ordered_cols]
+
+        connection_mock.cursor.return_value = self.cnx.cursor()
+        connection_mock.commit = self.cnx.commit
+        mock_sql.return_value = connection_mock
+
+        update(ordered_df, self.logger)
+
+        response = Epidata.rvdss(geo_type='province',
+                                 time_values= [202435, 202436],
+                                 geo_value = ['nt','nu'])
+
+        self.assertEqual(response,expected_response2)
+
+
+    # request by issue
+    with self.subTest(name='issue request'):
+        response = Epidata.rvdss(geo_type='province',
+                                 time_values= [202435, 202436],
+                                 geo_value = ['nl','nb'],
+                                 issues = 20250227)
+
+        self.assertEqual(response,expected_response)
+
+
+    # check requesting lists vs single values
+    with self.subTest(name='duplicate aquisition'):
+        # * with geo_value, single geo_type, time_value, issue
+        connection_mock.cursor.return_value = self.cnx.cursor()
+        connection_mock.commit = self.cnx.commit
+        mock_sql.return_value = connection_mock
+
+        update(detection_subset3, self.logger)
+
+        response = Epidata.rvdss(geo_type='province',
+                                 time_values= [202435, 202436],
+                                 geo_value = "*",
+                                 issues = 20250227)
+
+        response2 = Epidata.rvdss(geo_type='lab',
+                                 time_values= 202435,
+                                 geo_value = 'ouest du québec',
+                                 issues = [20250220,20250227])
+
+        self.assertEqual(response,expected_response)
+        self.assertEqual(response2,expected_response3)
+
+
+

integrations/server/test_rvdss.py

@@ -0,0 +1,74 @@
+# first party
+from delphi.epidata.common.integration_test_base_class import DelphiTestBase
+
+
+class rvdssTest(DelphiTestBase):
+    """Basic integration tests for rvdss endpoint."""
+
+    def localSetUp(self):
+        self.truncate_tables_list = ["rvdss"]
+
+    def test_rvdss_repiratory_detections(self):
+        """Basic integration test for rvdss endpoint"""
+        self.cur.execute(
+            "INSERT INTO `rvdss` (`epiweek`, `time_value`,`time_type`, `issue`, `geo_type`, `geo_value`, `sarscov2_tests`, `sarscov2_positive_tests`, `sarscov2_pct_positive`, `flu_tests`, `flu_positive_tests`, `flu_pct_positive`, `fluah1n1pdm09_positive_tests`, `fluah3_positive_tests`, `fluauns_positive_tests`, `flua_positive_tests`, `flua_tests`, `flua_pct_positive`, `flub_positive_tests`, `flub_tests`, `flub_pct_positive`, `rsv_tests`, `rsv_positive_tests`, `rsv_pct_positive`, `hpiv_tests`, `hpiv1_positive_tests`, `hpiv2_positive_tests`, `hpiv3_positive_tests`, `hpiv4_positive_tests`, `hpivother_positive_tests`, `hpiv_positive_tests`, `hpiv_pct_positive`, `adv_tests`, `adv_positive_tests`, `adv_pct_positive`, `hmpv_tests`, `hmpv_positive_tests`, `hmpv_pct_positive`, `evrv_tests`, `evrv_positive_tests`, `evrv_pct_positive`, `hcov_tests`, `hcov_positive_tests`, `hcov_pct_positive`, `year`) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
+            (201212,20120412,"week",20120417,'province','on',10.0,1.0,10.0,20.0, 10.0, 50.0, 1.0, 1.0, 1.0, 5.0, 20.0, 25.0, 4.0, 20.0, 20.0, 30.0, 3.0, 10.0, 40.0, 2.0, 2.0, 2.0, 1.0, 1.0, 8.0, 20.0, 40.0, 16.0, 40.0, 10.0, 2.0, 20.0, 1.0, 0.0, 0.0, 24.0, 3.0, 12.5, 2012),
+        )
+        self.cnx.commit()
+
+        response = self.epidata_client.rvdss(geo_type="province", time_values = 201212,geo_value="on")
+        self.assertEqual(
+            response,
+            {
+                "epidata": [
+                    { "geo_type":"province",
+                      "geo_value":"on",
+                      "time_type":"week",
+                      "epiweek":201212,
+                      "time_value":20120412,
+                      "issue":20120417,
+                      "year":2012,
+                      "adv_pct_positive":40.0,
+                      "adv_positive_tests":16.0,
+                      "adv_tests":40.0,
+                      "evrv_pct_positive":0.0,
+                      "evrv_positive_tests":0.0,
+                      "evrv_tests":1.0,
+                      "flu_pct_positive":50.0,
+                      "flu_positive_tests":10.0,
+                      "flu_tests":20.0,
+                      "flua_pct_positive":25.0,
+                      "flua_positive_tests":5.0,
+                      "flua_tests":20.0,
+                      "fluah1n1pdm09_positive_tests":1.0,
+                      "fluah3_positive_tests":1.0,
+                      "fluauns_positive_tests":1.0,
+                      "flub_pct_positive":20.0,
+                      "flub_positive_tests":4.0,
+                      "flub_tests":20.0,
+                      "hcov_pct_positive":12.5,
+                      "hcov_positive_tests":3.0,
+                      "hcov_tests":24.0, 
+                      "hmpv_pct_positive":20.0,
+                      "hmpv_positive_tests":2.0,
+                      "hmpv_tests":10.0,
+                      "hpiv1_positive_tests":2.0,
+                      "hpiv2_positive_tests":2.0,
+                      "hpiv3_positive_tests":2.0,
+                      "hpiv4_positive_tests":1.0,
+                      "hpiv_pct_positive":20.0,
+                      "hpiv_positive_tests":8.0,
+                      "hpiv_tests":40.0,
+                      "hpivother_positive_tests":1.0,
+                      "rsv_pct_positive":10.0,
+                      "rsv_positive_tests":3.0,
+                      "rsv_tests":30.0,
+                      "sarscov2_pct_positive":10.0,
+                      "sarscov2_positive_tests":1.0,
+                      "sarscov2_tests":10.0
+                    }
+                ],
+                "result": 1,
+                "message": "success",
+            },
+        )

requirements.api.txt

@@ -9,11 +9,13 @@ pandas==1.2.3
 python-dotenv==0.15.0
 pyyaml
 redis==3.5.3
+regex
 requests==2.32.4
 scipy==1.10.0
 sentry-sdk[flask]
 SQLAlchemy==1.4.40
 structlog==22.1.0
 tenacity==7.0.0
 typing-extensions
+unidecode
 werkzeug==3.0.6

requirements.dev.txt

@@ -19,3 +19,6 @@ selenium==4.7.2
 sqlalchemy-stubs>=0.3
 tenacity==7.0.0
 xlrd==2.0.1
+bs4
+mock
+requests_file