Merge pull request #613 from cmu-delphi/change-hhs-nation

Add nation and HHS to CHC

Author: krivard

ansible/templates/changehc-params-prod.json.j2

@@ -17,7 +17,7 @@
   "n_waiting_days": 3,
   "se": false,
   "parallel": false,
-  "geos": ["state", "msa", "hrr", "county"],
+  "geos": ["state", "msa", "hrr", "county", "hhs", "nation"],
   "weekday": [true, false],
   "types": ["covid","cli"],
   "wip_signal": "",

changehc/delphi_changehc/config.py

@@ -68,8 +68,12 @@ class Constants:
     NUM_HRRS = 308
     NUM_MSAS = 392 + 52  # MSA + States
     NUM_STATES = 52  # including DC and PR
+    NUM_NATIONS = 1
+    NUM_HHSS = 10
 
     MAX_GEO = {"county": NUM_COUNTIES,
                "hrr": NUM_HRRS,
                "msa": NUM_MSAS,
-               "state": NUM_STATES}
+               "state": NUM_STATES,
+               "nation": NUM_NATIONS,
+               "hhs": NUM_HHSS}

changehc/delphi_changehc/update_sensor.py

@@ -91,7 +91,7 @@ def __init__(self,
             startdate: first sensor date (YYYY-mm-dd)
             enddate: last sensor date (YYYY-mm-dd)
             dropdate: data drop date (YYYY-mm-dd)
-            geo: geographic resolution, one of ["county", "state", "msa", "hrr"]
+            geo: geographic resolution, one of ["county", "state", "msa", "hrr", "hhs", "nation"]
             parallel: boolean to run the sensor update in parallel
             weekday: boolean to adjust for weekday effects
             numtype: type of count data used, one of ["covid", "cli"]
@@ -104,9 +104,8 @@ def __init__(self,
                 ), f"not enough data to produce estimates starting {self.startdate}"
         assert self.startdate < self.enddate, "start date >= end date"
         assert self.enddate <= self.dropdate, "end date > drop date"
-        assert geo in ['county', 'state', 'msa', 'hrr'],\
-            f"{geo} is invalid, pick one of 'county', 'state', 'msa', 'hrr'"
-        self.geo, self.parallel, self.weekday, self.numtype, self.se = geo.lower(), parallel, weekday, numtype, se
+        self.geo, self.parallel, self.weekday, self.numtype, self.se = geo.lower(), parallel, \
+                                                                       weekday, numtype, se
 
         # output file naming
         if self.numtype == "covid":
@@ -145,10 +144,9 @@ def geo_reindex(self, data):
         # get right geography
         geo = self.geo
         gmpr = GeoMapper()
-        if geo not in {"county", "state", "msa", "hrr"}:
-            logging.error("{0} is invalid, pick one of 'county', 'state', 'msa', 'hrr'".format(
-                geo
-            ))
+        if geo not in {"county", "state", "msa", "hrr", "nation", "hhs"}:
+            logging.error("{0} is invalid, pick one of 'county', "
+                          "'state', 'msa', 'hrr', 'hss','nation'".format(geo))
             return False
         if geo == "county":
             data_frame = gmpr.fips_to_megacounty(data,
@@ -158,10 +156,8 @@ def geo_reindex(self, data):
                                                  mega_col=geo)
         elif geo == "state":
             data_frame = gmpr.replace_geocode(data, "fips", "state_id", new_col="state")
-        elif geo == "msa":
-            data_frame = gmpr.replace_geocode(data, "fips", "msa")
-        elif geo == "hrr":
-            data_frame = gmpr.replace_geocode(data, "fips", "hrr")
+        else:
+            data_frame = gmpr.replace_geocode(data, "fips", geo)
 
         unique_geo_ids = pd.unique(data_frame[geo])
         data_frame.set_index([geo, Config.DATE_COL],inplace=True)

changehc/params.json.template

@@ -17,7 +17,7 @@
   "n_waiting_days": 3,
   "se": false,
   "parallel": false,
-  "geos": ["state", "msa", "hrr", "county"],
+  "geos": ["state", "msa", "hrr", "county", "nation", "hhs"],
   "weekday": [true, false],
   "types": ["covid","cli"],
   "wip_signal": "",

changehc/tests/params.json.template

@@ -11,7 +11,7 @@
   "n_waiting_days": 3,
   "se": false,
   "parallel": false,
-  "geos": ["state", "msa", "hrr", "county"],
+  "geos": ["state", "msa", "hrr", "county", "nation", "hhs"],
   "weekday": [true, false],
   "wip_signal": "",
   "aws_credentials": {

changehc/tests/test_update_sensor.py

@@ -36,7 +36,7 @@ class TestCHCSensorUpdator:
     prefix = "foo"
     small_test_data = pd.DataFrame({
         "num": [0, 100, 200, 300, 400, 500, 600, 100, 200, 300, 400, 500, 600],
-        "fips": [1.0] * 7 + [2.0] * 6,
+        "fips": ['01001'] * 7 + ['04007'] * 6,
         "den": [1000] * 7 + [2000] * 6,
         "date": [pd.Timestamp(f'03-{i}-2020') for i in range(1, 14)]}).set_index(["fips","date"])
 
@@ -64,20 +64,21 @@ def test_shift_dates(self):
 
     def test_geo_reindex(self):
         """Tests that the geo reindexer changes the geographic resolution."""
-        su_inst = CHCSensorUpdator(
-            "02-01-2020",
-            "06-01-2020",
-            "06-12-2020",
-            'county',
-            self.parallel,
-            self.weekday,
-            self.numtype,
-            self.se
-        )
-        su_inst.shift_dates()
-        data_frame = su_inst.geo_reindex(self.small_test_data.reset_index())
-        assert data_frame.shape[0] == 2*len(su_inst.fit_dates)
-        assert (data_frame.sum() == (4200,19000)).all()
+        for geo, multiple in [("nation", 1), ("county", 2), ("hhs", 2)]:
+            su_inst = CHCSensorUpdator(
+                "02-01-2020",
+                "06-01-2020",
+                "06-12-2020",
+                geo,
+                self.parallel,
+                self.weekday,
+                self.numtype,
+                self.se
+            )
+            su_inst.shift_dates()
+            data_frame = su_inst.geo_reindex(self.small_test_data.reset_index())
+            assert data_frame.shape[0] == multiple*len(su_inst.fit_dates)
+            assert (data_frame.sum() == (4200,19000)).all()
 
     def test_update_sensor(self):
         """Tests that the sensors are properly updated."""