Propagate geo refactor into safegraph

.gitignore

@@ -126,3 +126,7 @@ venv.bak/
 .retry
 .indicators-ansible-vault-pass
 indicators-ansible-vault-pass
+
+# testing_utils
+testing_utils/cache
+testing_utils/*.csv

_delphi_utils_python/.pylintrc

@@ -1,8 +1,21 @@
-[DESIGN]
 
-min-public-methods=1
+[MESSAGES CONTROL]
 
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
 
-[MESSAGES CONTROL]
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
 
-disable=R0801, C0330, E1101, E0611, C0114, C0116, C0103, R0913, R0914, W0702
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

_delphi_utils_python/data_proc/geomap/geo_data_proc.py

@@ -217,6 +217,12 @@ def create_jhu_uid_fips_crosswalk():
             {"jhu_uid": "63072999", "fips": "72000", "weight": 1.0},
         ]
     )
+    cruise_ships = pd.DataFrame(
+        [
+            {"jhu_uid": "84088888", "fips": "88888", "weight": 1.0},
+            {"jhu_uid": "84099999", "fips": "99999", "weight": 1.0},
+        ]
+    )
 
     jhu_df = (
         pd.read_csv(JHU_FIPS_URL, dtype={"UID": str, "FIPS": str})
@@ -234,7 +240,7 @@ def create_jhu_uid_fips_crosswalk():
     # Drop the JHU UIDs that were hand-modified
     dup_ind = jhu_df["jhu_uid"].isin(
         pd.concat(
-            [hand_additions, unassigned_states, out_of_state, puerto_rico_unassigned]
+            [hand_additions, unassigned_states, out_of_state, puerto_rico_unassigned, cruise_ships]
         )["jhu_uid"].values
     )
     jhu_df.drop(jhu_df.index[dup_ind], inplace=True)
@@ -391,6 +397,7 @@ def create_fips_population_table():
     df_pr = df_pr.groupby("fips").sum().reset_index()
     df_pr = df_pr[~df_pr["fips"].isin(census_pop["fips"])]
     census_pop_pr = pd.concat([census_pop, df_pr])
+
     census_pop_pr.to_csv(join(OUTPUT_DIR, FIPS_POPULATION_OUT_FILENAME), index=False)
 
 

_delphi_utils_python/delphi_utils/data/jhu_uid_fips_table.csv

@@ -82,8 +82,6 @@ jhu_uid,fips,weight
 63072149,72149,1.0
 63072151,72151,1.0
 63072153,72153,1.0
-84088888,88888,1.0
-84099999,99999,1.0
 84000001,01000,1.0
 84000002,02000,1.0
 84000004,04000,1.0

_delphi_utils_python/delphi_utils/geomap.py

@@ -413,9 +413,10 @@ def replace_geocode(
             df = df.groupby([new_col]).sum().reset_index()
         return df
 
-    def add_population_column(self, data, geocode_type, geocode_col=None):
+    def add_population_column(self, data, geocode_type, geocode_col=None, dropna=True):
         """
-        Appends a population column to a dateframe, based on the FIPS or ZIP code.
+        Appends a population column to a dataframe, based on the FIPS or ZIP code. If no
+        dataframe is provided, the full crosswalk from geocode to population is returned.
 
         Parameters
         ---------
@@ -433,24 +434,26 @@ def add_population_column(self, data, geocode_type, geocode_col=None):
             A dataframe with a population column appended.
         """
         geocode_col = geocode_type if geocode_col is None else geocode_col
+        data = data.copy()
 
         if geocode_type not in ["fips", "zip"]:
             raise ValueError(
                 "Only fips and zip geocodes supported. \
                 For other codes, aggregate those."
             )
 
+        pop_df = self._load_crosswalk(from_code=geocode_type, to_code="pop")
+
         if not is_string_dtype(data[geocode_col]):
             data[geocode_col] = data[geocode_col].astype(str).str.zfill(5)
 
-        pop_df = self._load_crosswalk(from_code=geocode_type, to_code="pop")
-
+        merge_type = "inner" if dropna else "left"
         data_with_pop = (
-            data.copy()
-            .merge(pop_df, left_on=geocode_col, right_on=geocode_type, how="inner")
+            data
+            .merge(pop_df, left_on=geocode_col, right_on=geocode_type, how=merge_type)
             .rename(columns={"pop": "population"})
         )
-        data_with_pop["population"] = data_with_pop["population"].astype(int)
+
         return data_with_pop
 
     @staticmethod

_template_python/.pylintrc

@@ -0,0 +1,21 @@
+
+[MESSAGES CONTROL]
+
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
+
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
+
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

cdc_covidnet/.pylintrc

@@ -0,0 +1,21 @@
+
+[MESSAGES CONTROL]
+
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
+
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
+
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

changehc/.pylintrc

@@ -0,0 +1,21 @@
+
+[MESSAGES CONTROL]
+
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
+
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
+
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

claims_hosp/.pylintrc

@@ -0,0 +1,21 @@
+
+[MESSAGES CONTROL]
+
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
+
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
+
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

combo_cases_and_deaths/.pylintrc

@@ -0,0 +1,21 @@
+
+[MESSAGES CONTROL]
+
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
+
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
+
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

emr_hosp/.pylintrc

@@ -0,0 +1,21 @@
+
+[MESSAGES CONTROL]
+
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
+
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
+
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

google_health/.pylintrc

@@ -1,8 +1,21 @@
-[DESIGN]
 
-min-public-methods=1
+[MESSAGES CONTROL]
 
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
 
-[MESSAGES CONTROL]
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
 
-disable=R0801, C0330, E1101, E0611, C0114, C0116, C0103, R0913, R0914, W0702
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

google_health/delphi_google_health/pull_api.py

@@ -77,7 +77,7 @@ def query(self, start_date: str, end_date: str, geo_id, dma=False) -> dict:
             start_date,
             end_date,
         )
-        data = self.service.getTimelinesForHealth(**params).execute()
+        data = self.service.getTimelinesForHealth(**params).execute() #pylint: disable=no-member
 
         return data
 

google_health/delphi_google_health/smooth.py

@@ -60,11 +60,12 @@ def _left_gauss_linear(s: np.ndarray, h=10, impute=False, minval=None) -> np.nda
 
     n = len(s)
     t = np.zeros_like(s, dtype=np.float64)
-    X = np.vstack([np.ones(n), np.arange(n)]).T
+    X = np.vstack([np.ones(n), np.arange(n)]).T #pylint: disable=invalid-name
     for idx in range(n):
         wts = np.exp(-((np.arange(idx + 1) - idx) ** 2) / (h ** 2))
-        XwX = np.dot(X[: (idx + 1), :].T * wts, X[: (idx + 1), :])
-        Xwy = np.dot(X[: (idx + 1), :].T * wts, s[: (idx + 1)].reshape(-1, 1))
+        XwX = np.dot(X[: (idx + 1), :].T * wts, X[: (idx + 1), :]) #pylint: disable=invalid-name
+        Xwy = np.dot(X[: (idx + 1), :].T * wts, #pylint: disable=invalid-name
+                     s[: (idx + 1)].reshape(-1, 1))
         try:
             beta = np.linalg.solve(XwX, Xwy)
             t[idx] = np.dot(X[: (idx + 1), :], beta)[-1]

jhu/.pylintrc

@@ -1,8 +1,21 @@
-[DESIGN]
 
-min-public-methods=1
+[MESSAGES CONTROL]
 
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
 
-[MESSAGES CONTROL]
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
 
-disable=R0801, C0330, E1101, E0611, C0114, C0116, C0103, R0913, R0914, W0702
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

jhu/delphi_jhu/geo.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
+"""Functions for translating between geocodes."""
 import pandas as pd
 from delphi_utils import GeoMapper
 
+VALID_GEO_RES = ("county", "state", "msa", "hrr")
 INCIDENCE_BASE = 100000
 
 def geo_map(df: pd.DataFrame, geo_res: str):
@@ -27,14 +29,19 @@ def geo_map(df: pd.DataFrame, geo_res: str):
         Columns: geo_id, timestamp, ...
     """
     df = df.copy()
-    VALID_GEO_RES = ("county", "state", "msa", "hrr")
     if geo_res not in VALID_GEO_RES:
         raise ValueError(f"geo_res must be one of {VALID_GEO_RES}")
 
     gmpr = GeoMapper()
     if geo_res == "county":
         df.rename(columns={'fips': 'geo_id'}, inplace=True)
     elif geo_res == "state":
+        df = df.set_index("fips")
+        # Zero out the state FIPS population to avoid double counting.
+        state_fips_codes = {str(x).zfill(2) + "000" for x in range(1,73)}
+        subset_state_fips_codes = set(df.index.values) & state_fips_codes
+        df.loc[subset_state_fips_codes, "population"] = 0
+        df = df.reset_index()
         df = gmpr.replace_geocode(df, "fips", "state_id", new_col="geo_id", date_col="timestamp")
     else:
         df = gmpr.replace_geocode(df, "fips", geo_res, new_col="geo_id", date_col="timestamp")