Skip to content

Commit 4e5a051

Browse files
cchuongcchuong
authored
Merge pull request #1561 from cmu-delphi/ndefries/rvdss-tests
Add unit tests to `rvdss`
2 parents 30f3df6 + 277c44b commit 4e5a051

33 files changed

+34614
-380
lines changed

‎integrations/acquisition/rvdss/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
import sys
2+
import os
3+
4+
sys.path.append(os.getcwd())

‎integrations/acquisition/rvdss/test_scenarios.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
"""Integration tests for acquisition of rvdss data."""
2+
# standard library
3+
import unittest
4+
from unittest.mock import MagicMock
5+
6+
# first party
7+
from delphi.epidata.client.delphi_epidata import Epidata
8+
from delphi.epidata.acquisition.rvdss.database import update
9+
import delphi.operations.secrets as secrets
10+
11+
# third party
12+
import mysql.connector
13+
14+
# py3tester coverage target (equivalent to `import *`)
15+
# __test_target__ = 'delphi.epidata.acquisition.covid_hosp.facility.update'
16+
17+
NEWLINE="\n"
18+
19+
class AcquisitionTests(unittest.TestCase):
20+
21+
def setUp(self):
22+
"""Perform per-test setup."""
23+
24+
# configure test data
25+
# self.test_utils = UnitTestUtils(__file__)
26+
27+
# use the local instance of the Epidata API
28+
Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
29+
Epidata.auth = ('epidata', 'key')
30+
31+
# use the local instance of the epidata database
32+
secrets.db.host = 'delphi_database_epidata'
33+
secrets.db.epi = ('user', 'pass')
34+
35+
# clear relevant tables
36+
u, p = secrets.db.epi
37+
cnx = mysql.connector.connect(user=u, password=p, database="epidata")
38+
cur = cnx.cursor()
39+
40+
cur.execute('truncate table rvdss_repiratory_detections')
41+
cur.execute('truncate table rvdss_pct_positive')
42+
cur.execute('truncate table rvdss_detections_counts')
43+
cur.execute('delete from api_user')
44+
cur.execute('insert into api_user(api_key, email) values ("key", "emai")')
45+
46+
def test_rvdss_repiratory_detections(self):
47+
# make sure the data does not yet exist
48+
with self.subTest(name='no data yet'):
49+
response = Epidata.rvdss_repiratory_detections(
50+
'450822', Epidata.range(20200101, 20210101))
51+
self.assertEqual(response['result'], -2, response)
52+
53+
# acquire sample data into local database
54+
with self.subTest(name='first acquisition'):
55+
acquired = Update.run(network=mock_network)
56+
self.assertTrue(acquired)
57+
58+
# make sure the data now exists
59+
with self.subTest(name='initial data checks'):
60+
expected_spotchecks = {
61+
"hospital_pk": "450822",
62+
"collection_week": 20201030,
63+
"publication_date": 20210315,
64+
"previous_day_total_ed_visits_7_day_sum": 536,
65+
"total_personnel_covid_vaccinated_doses_all_7_day_sum": 18,
66+
"total_beds_7_day_avg": 69.3,
67+
"previous_day_admission_influenza_confirmed_7_day_sum": -999999
68+
}
69+
response = Epidata.covid_hosp_facility(
70+
'450822', Epidata.range(20200101, 20210101))
71+
self.assertEqual(response['result'], 1)
72+
self.assertEqual(len(response['epidata']), 2)
73+
row = response['epidata'][0]
74+
for k,v in expected_spotchecks.items():
75+
self.assertTrue(
76+
k in row,
77+
f"no '{k}' in row:\n{NEWLINE.join(sorted(row.keys()))}"
78+
)
79+
if isinstance(v, float):
80+
self.assertAlmostEqual(row[k], v, f"row[{k}] is {row[k]} not {v}")
81+
else:
82+
self.assertEqual(row[k], v, f"row[{k}] is {row[k]} not {v}")
83+
84+
# expect 113 fields per row (114 database columns, except `id`)
85+
self.assertEqual(len(row), 113)
86+
87+
# re-acquisition of the same dataset should be a no-op
88+
with self.subTest(name='second acquisition'):
89+
acquired = Update.run(network=mock_network)
90+
self.assertFalse(acquired)
91+
92+
# make sure the data still exists
93+
with self.subTest(name='final data checks'):
94+
response = Epidata.covid_hosp_facility(
95+
'450822', Epidata.range(20200101, 20210101))
96+
self.assertEqual(response['result'], 1)
97+
self.assertEqual(len(response['epidata']), 2)
98+
99+

‎integrations/server/test_rvdss.py‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
# first party
2+
from delphi.epidata.common.integration_test_base_class import DelphiTestBase
3+
4+
5+
class rvdssTest(DelphiTestBase):
6+
"""Basic integration tests for rvdss endpoint."""
7+
8+
def localSetUp(self):
9+
self.truncate_tables_list = ["rvdss_repiratory_detections",
10+
"rvdss_pct_positive",
11+
"rvdss_detections_counts"]
12+
13+
def test_rvdss_repiratory_detections(self):
14+
"""Basic integration test for rvdss_repiratory_detections endpoint"""
15+
self.cur.execute(
16+
"INSERT INTO `rvdss_repiratory_detections`(`epiweek`, `time_value`, `issue`, `geo_type`, `geo_value`, `sarscov2_tests`, `sarscov2_positive_tests`, `flu_tests`, `flu_positive_tests`, `fluah1n1pdm09_positive_tests`, `fluah3_positive_tests`, `fluauns_positive_tests`, `flua_positive_tests`, `flub_positive_tests`, `rsv_tests`, `rsv_positive_tests`, `hpiv_tests`, `hpiv1_positive_tests`, `hpiv2_positive_tests`, `hpiv3_positive_tests`, `hpiv4_positive_tests`, `hpivother_positive_tests`, `adv_tests`, `adv_positive_tests`, `hmpv_tests`, `hmpv_positive_tests`, `evrv_tests`, `evrv_positive_tests`, `hcov_tests`, `hcov_positive_tests`, `week`, `weekorder`, `year`) VALUES(%(epiweek)s, %(time_value)s, %(issue)s, %(geo_type)s, %(geo_value)s, %(sarscov2_tests)s, %(sarscov2_positive_tests)s, %(flu_tests)s, %(flu_positive_tests)s, %(fluah1n1pdm09_positive_tests)s, %(fluah3_positive_tests)s, %(fluauns_positive_tests)s, %(flua_positive_tests)s, %(flub_positive_tests)s, %(rsv_tests)s, %(rsv_positive_tests)s, %(hpiv_tests)s, %(hpiv1_positive_tests)s, %(hpiv2_positive_tests)s, %(hpiv3_positive_tests)s, %(hpiv4_positive_tests)s, %(hpivother_positive_tests)s, %(adv_tests)s, %(adv_positive_tests)s, %(hmpv_tests)s, %(hmpv_positive_tests)s, %(evrv_tests)s, %(evrv_positive_tests)s, %(hcov_tests)s, %(hcov_positive_tests)s, %(week)s, %(weekorder)s, %(year)s)",
17+
("201212", "2012-04-12", "2012-04-17", "region","on", "10", "1", "9", "1", "0", "0", "2", "3", "1", "8", "2", "7", "1", "1", "1", "1","1", "6", "5", "100", "13", "92", "9", "167", "52", "12", "34", "2012"),
18+
)
19+
self.cnx.commit()
20+
response = self.epidata_client.rvdss_repiratory_detections(epiweeks=201212, geo_value="on")
21+
self.assertEqual(
22+
response,
23+
{
24+
"epidata": [
25+
{'epiweek':201212,
26+
'time_value':"2012-04-12",
27+
'issue':"2012-04-17",
28+
'geo_type':"region",
29+
'geo_value':"on",
30+
'sarscov2_tests':10,
31+
'sarscov2_positive_tests':1,
32+
'flu_tests':9,
33+
'flu_positive_tests':1,
34+
'fluah1n1pdm09_positive_tests':0,
35+
'fluah3_positive_tests':0,
36+
'fluauns_positive_tests':2,
37+
'flua_positive_tests':3,
38+
'flub_positive_tests':1,
39+
'rsv_tests':8,
40+
'rsv_positive_tests':2,
41+
'hpiv_tests':8,
42+
'hpiv1_positive_tests':1,
43+
'hpiv2_positive_tests':1,
44+
'hpiv3_positive_tests':1,
45+
'hpiv4_positive_tests':1,
46+
'hpivother_positive_tests':1,
47+
'adv_tests':6,
48+
'adv_positive_tests':5,
49+
'hmpv_tests':100,
50+
'hmpv_positive_tests':13,
51+
'evrv_tests':92,
52+
'evrv_positive_tests':9,
53+
'hcov_tests':167,
54+
'hcov_positive_tests':52,
55+
'week':12,
56+
'weekorder':34,
57+
'year':2012
58+
}
59+
],
60+
"result": 1,
61+
"message": "success",
62+
},
63+
)
64+
65+
def test_rvdss_pct_positive(self):
66+
"""Basic integration test for rvdss_pct_positive endpoint"""
67+
self.cur.execute(
68+
"INSERT INTO rvdss_pct_positive (`epiweek`, `time_value`, `issue`, `geo_type`, `geo_value`, `evrv_pct_positive`, `evrv_tests`, `evrv_positive_tests`, `hpiv_pct_positive`, `hpiv_tests`, `hpiv_positive_tests`, `adv_pct_positive`, `adv_tests`, `adv_positive_tests`,`hcov_pct_positive`, `hcov_tests`, `hcov_positive_tests`, `flua_pct_positive`, `flub_pct_positive`, `flu_tests`, `flua_positive_tests`, `flua_tests`, `flub_tests`, `flub_positive_tests`, `flu_positive_tests`, `flu_pct_positive`, `hmpv_pct_positive`, `hmpv_tests`, `hmpv_positive_tests`, `rsv_pct_positive`, `rsv_tests`, `rsv_positive_tests`, `sarscov2_pct_positive`, `sarscov2_tests`, `sarscov2_positive_tests`, `region`, `week`, `weekorder`, `year`) VALUES (%(epiweek)s, %(time_value)s, %(issue)s, %(geo_type)s, %(geo_value)s, %(evrv_pct_positive)s, %(evrv_tests)s, %(evrv_positive_tests)s, %(hpiv_pct_positive)s, %(hpiv_tests)s, %(hpiv_positive_tests)s, %(adv_pct_positive)s, %(adv_tests)s, %(hcov_pct_positive)s, %(hcov_tests)s, %(hcov_positive_tests)s, %(flua_pct_positive)s, %(flub_pct_positive)s, %(flu_tests)s, %(flua_positive_tests)s, %(flua_tests)s, %(flub_tests)s, %(flub_positive_tests)s, %(flu_positive_tests)s, %(flu_pct_positive)s, %(hmpv_pct_positive)s, %(hmpv_tests)s, %(hmpv_positive_tests)s, %(rsv_pct_positive)s, %(rsv_tests)s, %(rsv_positive_tests)s, %(sarscov2_pct_positive)s, %(sarscov2_tests)s, %(sarscov2_positive_tests)s, %(region)s, %(week)s, %(weekorder)s, %(year)s)",
69+
("201212", "2012-04-12", "2012-04-17", "region","on","0.1","10","1","0.1","10","1","0.1","10","1","0.1","10","1","0.05","0.05","100", "10", "100","100", "10", "20", "0.1","0.1","10","1","0.1","10","1","0.1","10","1","on","12","34","2012")
70+
)
71+
self.cnx.commit()
72+
response = self.epidata_client.rvdss_pct_positive(epiweeks=201212, geo_value="on")
73+
self.assertEqual(
74+
response,
75+
{
76+
"epidata": [
77+
{'epiweek':201212,
78+
'time_value':"2012-04-12",
79+
'issue':"2012-04-17",
80+
'geo_type':"region",
81+
'geo_value':"on",
82+
'evrv_pct_positive':0.1,
83+
'evrv_tests':10,
84+
'evrv_positive_tests':1,
85+
'hpiv_pct_positive':0.1,
86+
'hpiv_tests':10,
87+
'hpiv_positive_tests':1,
88+
'adv_pct_positive':0.1,
89+
'adv_tests':10,
90+
'adv_positive_tests':1,
91+
'hcov_pct_positive':0.1,
92+
'hcov_tests':10,
93+
'hcov_positive_tests':1,
94+
'flua_pct_positive':0.05,
95+
'flub_pct_positive':0.05,
96+
'flu_tests':100,
97+
'flua_positive_tests':10,
98+
'flua_tests':100,
99+
'flub_tests':100,
100+
'flub_positive_tests':10,
101+
'flu_positive_tests':20,
102+
'flu_pct_positive':0.1,
103+
'hmpv_pct_positive':0.1,
104+
'hmpv_tests':10,
105+
'hmpv_positive_tests':1,
106+
'rsv_pct_positive':0.1,
107+
'rsv_tests':10,
108+
'rsv_positive_tests':1,
109+
'sarscov2_pct_positive':0.1,
110+
'sarscov2_tests':10,
111+
'sarscov2_positive_tests':1,
112+
'region':"on",
113+
'week':12,
114+
'weekorder':34,
115+
'year':2012
116+
}
117+
],
118+
"result": 1,
119+
"message": "success",
120+
},
121+
)
122+
123+
def test_rvdss_detections_counts(self):
124+
"""Basic integration test for rvdss_detections_counts endpoint"""
125+
self.cur.execute(
126+
"INSERT INTO rvdss_detections_counts (`epiweek`, `time_value`, `issue`, `geo_type`, `geo_value`, `hpiv_positive_tests`, `adv_positive_tests`, `hmpv_positive_tests`, `evrv_positive_tests`, `hcov_positive_tests`, `rsv_positive_tests`, `flu_positive_tests`) VALUES (%(epiweek)s, %(time_value)s, %(issue)s, %(geo_type)s, %(geo_value)s, %(hpiv_positive_tests)s, %(adv_positive_tests)s, %(hmpv_positive_tests)s, %(evrv_positive_tests)s, %(hcov_positive_tests)s, %(rsv_positive_tests)s, %(flu_positive_tests)s)",
127+
("201212", "2012-04-12", "2012-04-17", "nation","ca", "10", "9", "8", "7", "6", "5", "4"),
128+
)
129+
self.cnx.commit()
130+
response = self.epidata_client.rvdss_detections_counts(epiweeks=201212, geo_value="ca")
131+
self.assertEqual(
132+
response,
133+
{
134+
"epidata": [
135+
{'epiweek':201212,
136+
'time_value':"2012-04-12",
137+
'issue':"2012-04-17",
138+
'geo_type':"nation",
139+
'geo_value':"ca",
140+
'hpiv_positive_tests':10,
141+
'adv_positive_tests':9,
142+
'hmpv_positive_tests':8,
143+
'evrv_positive_tests':7,
144+
'hcov_positive_tests':6,
145+
'rsv_positive_tests':5,
146+
'flu_positive_tests':4
147+
}
148+
],
149+
"result": 1,
150+
"message": "success",
151+
},
152+
)

‎requirements.api.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@ pandas==1.2.3
1010
python-dotenv==0.15.0
1111
pyyaml
1212
redis==3.5.3
13+
regex
1314
requests==2.32.0
1415
scipy==1.10.0
1516
sentry-sdk[flask]
1617
SQLAlchemy==1.4.40
1718
structlog==22.1.0
1819
tenacity==7.0.0
1920
typing-extensions
21+
unidecode
2022
werkzeug==2.3.8

‎requirements.dev.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,6 @@ selenium==4.7.2
1919
sqlalchemy-stubs>=0.3
2020
tenacity==7.0.0
2121
xlrd==2.0.1
22+
bs4
23+
mock
24+
requests_file

‎src/acquisition/rvdss/constants.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@
5353
'prairies', 'pr', "british columbia",'bc',"territories",'terr',]
5454
NATION = ["canada","can",'ca',]
5555

56+
PROVINCES = ['nl','pe','ns','nb','qc','on','mb','sk','ab','bc','yt','nt','nu']
57+
5658
# Construct dashboard and data report URLS.
5759
DASHBOARD_BASE_URL = "https://health-infobase.canada.ca/src/data/respiratory-virus-detections/"
5860
DASHBOARD_W_DATE_URL = DASHBOARD_BASE_URL + "archive/{date}/"
@@ -115,3 +117,5 @@
115117

116118
UPDATE_DATES_FILE = "update_dates.txt"
117119
NOW = datetime.now()
120+
121+
LOGGER_FILENAME = "rvdss.log"

0 commit comments

Comments
 (0)