Merge branch 'main' into utils-ci

Author: chinandrew

.github/workflows/python-ci.yml

@@ -34,7 +34,7 @@ jobs:
       run: |
         make install
     - name: Lint
-      if: ${{ matrix.packages != 'changehc' && matrix.packages != 'claims_hosp' && matrix.packages != 'quidel' && matrix.packages != 'usafacts'}}
+      if: ${{ matrix.packages != 'claims_hosp' && matrix.packages != 'quidel' && matrix.packages != 'usafacts'}}
       run: |
         make lint
     - name: Test

_delphi_utils_python/delphi_utils/archive.py

@@ -588,11 +588,11 @@ def archive_exports(self, exported_files: Files) -> Tuple[Files, Files]:
     args = parser.parse_args()
     params = read_params()
     run_module(args.archive_type,
-               params.cache_dir,
-               params.export_dir,
-               aws_credentials=params.aws_credentials,
+               params["cache_dir"],
+               params["export_dir"],
+               aws_credentials=params["aws_credentials"],
                branch_name=args.branch_name,
-               bucket_name=params.bucket_name,
+               bucket_name=params["bucket_name"],
                commit_message=args.commit_message,
                commit_partial_success=args.commit_partial_success,
                indicator_prefix=args.indicator_prefix,

_delphi_utils_python/delphi_utils/smooth.py

@@ -150,9 +150,9 @@ def __init__(
         else:
             self.coeffs = None
 
-    def smooth(self,
-               signal: Union[np.ndarray, pd.Series],
-               impute_order=2) -> Union[np.ndarray, pd.Series]:
+    def smooth(
+        self, signal: Union[np.ndarray, pd.Series], impute_order=2
+    ) -> Union[np.ndarray, pd.Series]:
         """Apply a smoother to a signal.
 
         The major workhorse smoothing function. Imputes the nans and then applies
@@ -202,7 +202,7 @@ def smooth(self,
                 signal_smoothed = signal
 
         # Append the nans back, since we want to preserve length
-        signal_smoothed = np.hstack([np.nan*np.ones(ix), signal_smoothed])
+        signal_smoothed = np.hstack([np.nan * np.ones(ix), signal_smoothed])
         # Convert back to pandas if necessary
         if is_pandas_series:
             signal_smoothed = pd.Series(signal_smoothed)
@@ -297,21 +297,28 @@ def left_gauss_linear_smoother(self, signal):
             weights = np.exp(
                 -((np.arange(idx + 1) - idx) ** 2) / self.gaussian_bandwidth
             )
-            AwA = np.dot(A[: (idx + 1), :].T * weights, A[: (idx + 1), :])  # pylint: disable=invalid-name
+            AwA = np.dot(  # pylint: disable=invalid-name
+                A[: (idx + 1), :].T * weights, A[: (idx + 1), :]
+            )
             Awy = np.dot(  # pylint: disable=invalid-name
                 A[: (idx + 1), :].T * weights, signal[: (idx + 1)].reshape(-1, 1)
             )
             try:
                 beta = np.linalg.solve(AwA, Awy)
                 signal_smoothed[idx] = np.dot(A[: (idx + 1), :], beta)[-1]
             except np.linalg.LinAlgError:
-                signal_smoothed[idx] = signal[idx] if self.impute else np.nan  # pylint: disable=using-constant-test
+                signal_smoothed[idx] = (
+                    signal[idx]  # pylint: disable=using-constant-test
+                    if self.impute
+                    else np.nan
+                )
         if self.minval is not None:
             signal_smoothed[signal_smoothed <= self.minval] = self.minval
         return signal_smoothed
 
     def savgol_predict(self, signal, poly_fit_degree, nr):
         """Predict a single value using the savgol method.
+
         Fits a polynomial through the values given by the signal and returns the value
         of the polynomial at the right-most signal-value. More precisely, for a signal of length
         n, fits a poly_fit_degree polynomial through the points signal[-n+1+nr], signal[-n+2+nr],
@@ -473,7 +480,7 @@ def savgol_impute(self, signal, impute_order):
                 # imputation order is larger than the available data)
                 else:
                     signal_imputed[ix] = self.savgol_predict(
-                        signal_imputed[:ix], min(ix-1, impute_order), -1
+                        signal_imputed[:ix], min(ix - 1, impute_order), -1
                     )
             # Away from the boundary, use savgol fitting on a fixed window
             else:

ansible/templates/changehc-params-prod.json.j2

@@ -2,8 +2,8 @@
   "static_file_dir": "./static",
   "export_dir": "/common/covidcast/receiving/changehc",
   "cache_dir": "./cache",
-  "input_denom_file": "./tests/test_data/20200601_All_Outpatients_By_County.dat",
-  "input_covid_file": "./tests/test_data/20200601_Covid_Outpatients_By_County.dat",
+  "input_denom_file": null,
+  "input_covid_file": null,
   "start_date": "2020-02-01",
   "end_date": null,
   "drop_date": null,

changehc/.pylintrc

@@ -7,7 +7,9 @@ disable=logging-format-interpolation,
     # Allow pytest functions to be part of a class.
     no-self-use,
     # Allow pytest classes to have one test.
-    too-few-public-methods
+    too-few-public-methods,
+    # Ignore
+    R0903, C0301, R0914, C0103, W1203, E0611, R0902, R0913, W0105, W0611, W1401
 
 [BASIC]
 

changehc/delphi_changehc/constants.py

@@ -1,6 +1,6 @@
 """Registry for signal names and geo types"""
-SMOOTHED = "smoothed_cli"
-SMOOTHED_ADJ = "smoothed_adj_cli"
+SMOOTHED = "smoothed_outpatient_covid"
+SMOOTHED_ADJ = "smoothed_adj_outpatient_covid"
 SIGNALS = [SMOOTHED, SMOOTHED_ADJ]
 NA = "NA"
 HRR = "hrr"

changehc/delphi_changehc/run.py

@@ -26,27 +26,33 @@ def run_module():
 
     logging.basicConfig(level=logging.DEBUG)
 
-    ## download recent files from FTP server
-    logging.info("downloading recent files through SFTP")
-    download(params["cache_dir"], params["ftp_conn"])
-
-    ## get end date from input file
     # the filenames are expected to be in the format:
     # Denominator: "YYYYMMDD_All_Outpatients_By_County.dat.gz"
     # Numerator: "YYYYMMDD_Covid_Outpatients_By_County.dat.gz"
 
+    assert (params["input_denom_file"] is None) == (params["input_covid_file"] is None), \
+        "exactly one of denom and covid files are provided"
+
     if params["drop_date"] is None:
-        dropdate_denom = datetime.strptime(
-            Path(params["input_denom_file"]).name.split("_")[0], "%Y%m%d"
-        )
-
-        dropdate_covid = datetime.strptime(
-            Path(params["input_covid_file"]).name.split("_")[0], "%Y%m%d"
-        )
-        assert dropdate_denom == dropdate_covid, "different drop dates for data files"
-        dropdate_dt = dropdate_denom
+        # files are dropped about 8pm the day after the issue date
+        dropdate_dt = (datetime.now() - timedelta(days=1,hours=20))
+        dropdate_dt = dropdate_dt.replace(hour=0,minute=0,second=0,microsecond=0)
     else:
         dropdate_dt = datetime.strptime(params["drop_date"], "%Y-%m-%d")
+    filedate = dropdate_dt.strftime("%Y%m%d")
+
+    if params["input_denom_file"] is None:
+
+        ## download recent files from FTP server
+        logging.info("downloading recent files through SFTP")
+        download(params["cache_dir"], params["ftp_conn"])
+
+        input_denom_file = "%s/%s_All_Outpatients_By_County.dat.gz" % (params["cache_dir"],filedate)
+        input_covid_file = "%s/%s_Covid_Outpatients_By_County.dat.gz" % (params["cache_dir"],filedate)
+    else:
+        input_denom_file = params["input_denom_file"]
+        input_covid_file = params["input_covid_file"]
+
     dropdate = str(dropdate_dt.date())
 
     # range of estimates to produce
@@ -93,8 +99,8 @@ def run_module():
                 params["se"]
             )
             su_inst.update_sensor(
-                params["input_denom_file"],
-                params["input_covid_file"],
+                input_denom_file,
+                input_covid_file,
                 params["export_dir"]
             )
         logging.info("finished %s", geo)

changehc/params.json.template

@@ -2,8 +2,8 @@
   "static_file_dir": "./static",
   "export_dir": "./receiving",
   "cache_dir": "./cache",
-  "input_denom_file": "./tests/test_data/20200601_All_Outpatients_By_County.dat",
-  "input_covid_file": "./tests/test_data/20200601_Covid_Outpatients_By_County.dat",
+  "input_denom_file": null,
+  "input_covid_file": null,
   "start_date": "2020-02-01",
   "end_date": null,
   "drop_date": null,

changehc/tests/test_sensor.py

@@ -1,16 +1,14 @@
 # standard
-import pytest
 
-# third party
-from delphi_utils import read_params, Smoother
 import numpy as np
 import numpy.random as nr
 import pandas as pd
-
 # first party
 from delphi_changehc.config import Config
 from delphi_changehc.load_data import load_combined_data
 from delphi_changehc.sensor import CHCSensor
+# third party
+from delphi_utils import read_params
 
 CONFIG = Config()
 PARAMS = read_params()
@@ -21,11 +19,6 @@
 class TestLoadData:
     combined_data = load_combined_data(DENOM_FILEPATH, COVID_FILEPATH, DROP_DATE,
                                        "fips")
-    # change smoother window length for test data
-    CHCSensor.smoother = Smoother("savgol",
-                                  poly_fit_degree=1,
-                                  gaussian_bandwidth=Config.SMOOTHER_BANDWIDTH,
-                                  window_length=20)
 
     def test_backfill(self):
         num0 = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8], dtype=float).reshape(-1, 1)

jhu/DETAILS.md

@@ -34,90 +34,16 @@ discrete difference of `cumulative_counts`,  and assume that the
 problem, because there there is only one county with a nonzero
 `cumulative_count` on January 22nd, with a value of 1.
 
-For deriving `incidence`, we use the estimated 2019 county population values
-from the US Census Bureau.  https://www.census.gov/data/tables/time-series/demo/popest/2010s-counties-total.html
+For deriving `incidence`, we use the estimated 2019 county population estimates
+from the [US Census Bureau](https://www.census.gov/data/tables/time-series/demo/popest/2010s-counties-total.html).
 
 ## Exceptions
 
-At the County (FIPS) level, we report the data _exactly_ as JHU reports their
-data, to prevent confusing public consumers of the data.
-The visualization and modeling teams should take note of these exceptions.
-
-### New York City
-
-New York City comprises of five boroughs:
-
-|Borough Name       |County Name        |FIPS Code      |
-|-------------------|-------------------|---------------|
-|Manhattan          |New York County    |36061          |
-|The Bronx          |Bronx County       |36005          |
-|Brooklyn           |Kings County       |36047          |
-|Queens             |Queens County      |36081          |
-|Staten Island      |Richmond County    |36085          |
-
-**Data from all five boroughs are reported under New York County,
-FIPS Code 36061.**  The other four boroughs are included in the dataset
-and show up in our API, but they should be uniformly zero. (In our population 
-file under static folder, the population from all five boroughs are also 
-assigned to FIPS Code 36061 only. The populatio for the rest of the counties 
-are set to be 1.)
-
-All NYC counts are mapped to the MSA with CBSA ID 35620, which encompasses
-all five boroughs.  All NYC counts are mapped to HRR 303, which intersects
-all five boroughs (297 also intersects the Bronx, 301 also intersects
-Brooklyn and Queens, but absent additional information, I am leaving all
-counts in 303).
-
-### Kansas City, Missouri
-
-Kansas City intersects the following four counties, which themselves report
-confirmed case and deaths data:
-
-|County Name        |FIPS Code      |
-|-------------------|---------------|
-|Jackson County     |29095          |
-|Platte County      |29165          |
-|Cass County        |29037          |
-|Clay County        |29047          |
-
-**Data from Kansas City is given its own dedicated line, with FIPS
-code 70003.**  This is how JHU encodes their data.  However, the data in
-the four counties that Kansas City intersects is not necessarily zero.
-
-For the mapping to HRR and MSA, the counts for Kansas City are dispersed to
-these four counties in equal proportions.
-
-### Dukes and Nantucket Counties, Massachusetts
-
-**The counties of Dukes and Nantucket report their figures together,
-and we (like JHU) list them under FIPS Code 70002.**  Here are the FIPS codes
-for the individual counties:
-
-|County Name        |FIPS Code      |
-|-------------------|---------------|
-|Dukes County       |25007          |
-|Nantucket County   |25019          |
-
-For the mapping to HRR and MSA, the counts for Dukes and Nantucket are
-dispersed to the two counties in equal proportions.
-
-The data in the individual counties is expected to be zero.
-
-### Mismatched FIPS Codes
-
-Finally, there are two FIPS codes that were changed in 2015, leading to
-mismatch between us and JHU.  We report the data using the FIPS code used
-by JHU, again to promote consistency and avoid confusion by external users
-of the dataset.  For the mapping to MSA, HRR, these two counties are
-included properly.
-
-|County Name        |State          |"Our" FIPS         |JHU FIPS       |
-|-------------------|---------------|-------------------|---------------|
-|Oglala Lakota      |South Dakota   |46113              |46102          |
-|Kusilvak           |Alaska         |02270              |02158          |
-
-Documentation for the changes made by the US Census Bureau in 2015:
-https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes.html
+To prevent confusing public consumers of the data, we report the data as closely
+as possible to the way JHU reports their data, using the same County FIPS codes.
+Nonetheless, there are a few exceptions which should be of interest to the
+visualization and modeling teams. These exceptions can be found at the [JHU Delphi
+Epidata API documentation page](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/jhu-csse.html#geographical-exceptions).
 
 ## Negative incidence
 
@@ -129,26 +55,3 @@ to County Y, County X may have negative incidence.
 
 Because the MSA and HRR numbers are computed by taking population-weighted
 averages, the count data at those geographical levels may be non-integral.
-
-## Counties not in our canonical dataset
-
-Some FIPS codes do not appear as the primary FIPS for any ZIP code in our
-canonical `02_20_uszips.csv`; they appear in the `county` exported files, but
-for the MSA/HRR mapping, we disburse them equally to the counties with whom
-they appear as a secondary FIPS code.  The identification of such "secondary"
-FIPS codes are documented in `notebooks/create-mappings.ipynb`.  The full list
-of `secondary, [mapped]` is:
-
-```
-SECONDARY_FIPS = [   # generated by notebooks/create-mappings.ipynb
-	('51620', ['51093', '51175']),
-	('51685', ['51153']),
-	('28039', ['28059', '28041', '28131', '28045', '28059', '28109',
-                    '28047']),
-	('51690', ['51089', '51067']),
-	('51595', ['51081', '51025', '51175', '51183']),
-	('51600', ['51059', '51059', '51059']),
-	('51580', ['51005']),
-	('51678', ['51163']),
-    ]
-```