Merge pull request #212 from cmu-delphi/handle_wip_signal_ght

ght: standardizing signal names

Author: krivard

google_health/delphi_google_health/constants.py

@@ -0,0 +1,13 @@
+"""
+REGISTRY FOR SIGNAL NAMES, GEO TYPES
+"""
+RAW = "raw_search"
+SMOOTHED = "smoothed_search"
+STATE = "state"
+HRR = "hrr"
+MSA = "msa"
+DMA = "dma"
+
+SIGNALS = [RAW, SMOOTHED]
+GEO_TYPES = [STATE, HRR, MSA, DMA]
+

google_health/delphi_google_health/run.py

@@ -9,10 +9,11 @@
 import logging
 
 from delphi_utils import read_params
-
+import covidcast
 from .pull_api import GoogleHealthTrends, get_counts_states, get_counts_dma
 from .map_values import derived_counts_from_dma
 from .export import export_csv
+from .constants import *
 
 
 def run_module():
@@ -31,6 +32,7 @@ def run_module():
     static_dir = params["static_file_dir"]
     export_dir = params["export_dir"]
     cache_dir = params["cache_dir"]
+    wip_signal = params["wip_signal"]
 
     # if missing start_date, set to today (GMT) minus 5 days
     if start_date == "":
@@ -58,17 +60,71 @@ def run_module():
     )
     df_hrr, df_msa = derived_counts_from_dma(df_dma, static_dir=static_dir)
 
-    #  export each geographic region, with both smoothed and unsmoothed data
-    export_csv(df_state, "state", "raw_search", smooth=False, receiving_dir=export_dir)
-    export_csv(
-        df_state, "state", "smoothed_search", smooth=True, receiving_dir=export_dir
-    )
-
-    export_csv(df_dma, "dma", "raw_search", smooth=False, receiving_dir=export_dir)
-    export_csv(df_dma, "dma", "smoothed_search", smooth=True, receiving_dir=export_dir)
-
-    export_csv(df_hrr, "hrr", "raw_search", smooth=False, receiving_dir=export_dir)
-    export_csv(df_hrr, "hrr", "smoothed_search", smooth=True, receiving_dir=export_dir)
+    signal_names = add_prefix(SIGNALS, wip_signal, prefix)
+
+    for signal in signal_names:
+        if signal.endswith(SMOOTHED):
+            #  export each geographic region, with both smoothed and unsmoothed data
+            export_csv(df_state, STATE, signal, smooth=True, receiving_dir=export_dir)
+            export_csv(df_dma, DMA, signal, smooth=True, receiving_dir=export_dir)
+            export_csv(df_hrr, HRR, signal, smooth=True, receiving_dir=export_dir)
+            export_csv(df_msa, MSA, signal, smooth=True, receiving_dir=export_dir)
+        elif signal.endswith(RAW):
+            export_csv(df_state, STATE, signal, smooth=False, receiving_dir=export_dir)
+            export_csv(df_dma, DMA, signal, smooth=False, receiving_dir=export_dir)
+            export_csv(df_hrr, HRR, signal, smooth=False, receiving_dir=export_dir)
+            export_csv(df_msa, MSA, signal, smooth=False, receiving_dir=export_dir)
+
+
+def add_prefix(signal_names, wip_signal, prefix="wip_"):
+    """Adds prefix to signal if there is a WIP signal
+    Parameters
+    ----------
+    signal_names: List[str]
+        Names of signals to be exported
+    prefix : 'wip_'
+        prefix for new/non public signals
+    wip_signal : List[str] or bool
+        a list of wip signals: [], OR
+        all signals in the registry: True OR
+        only signals that have never been published: False
+    Returns
+    -------
+    List of signal names
+        wip/non wip signals for further computation
+    """
 
-    export_csv(df_msa, "msa", "raw_search", smooth=False, receiving_dir=export_dir)
-    export_csv(df_msa, "msa", "smoothed_search", smooth=True, receiving_dir=export_dir)
+    if wip_signal is True:
+        return [prefix + signal for signal in signal_names]
+    if isinstance(wip_signal, list):
+        make_wip = set(wip_signal)
+        return [
+            (prefix if signal in make_wip else "") + signal
+            for signal in signal_names
+        ]
+    if wip_signal in {False, ""}:
+        return [
+            signal if public_signal(signal)
+            else prefix + signal
+            for signal in signal_names
+        ]
+    raise ValueError("Supply True | False or '' or [] | list()")
+
+
+def public_signal(signal_):
+    """Checks if the signal name is already public using COVIDcast
+    Parameters
+    ----------
+    signal_ : str
+        Name of the signal
+    Returns
+    -------
+    bool
+        True if the signal is present
+        False if the signal is not present
+    """
+    epidata_df = covidcast.metadata()
+    for index in range(len(epidata_df)):
+        if epidata_df['signal'][index] == signal_:
+            return True
+    return False

google_health/params.json.template

@@ -5,4 +5,5 @@
   "start_date": "2020-01-05",
   "end_date": "",
   "ght_key": ""
+  "wip_signal" : ""
 }

google_health/setup.py

@@ -8,7 +8,8 @@
     "pytest",
     "pytest-cov",
     "pylint",
-    "delphi-utils"
+    "delphi-utils",
+    "covidcast"
 ]
 
 setup(

google_health/tests/params.json.template

@@ -4,5 +4,6 @@
   "cache_dir": "../cache",
   "start_date": "2020-02-11",
   "end_date": "2020-04-30",
-  "ght_key": ""
+  "ght_key": "",
+  "wip_signal": ""
 }

google_health/tests/receiving_test/20200419_dma_wip_raw_search.csv

@@ -0,0 +1,211 @@
+geo_id,val,se,sample_size
+500,0.00000000,NA,NA
+501,25.65377897,NA,NA
+502,334.67202142,NA,NA
+503,175.13134851,NA,NA
+504,25.48638218,NA,NA
+505,49.01260746,NA,NA
+506,12.96612475,NA,NA
+507,0.00000000,NA,NA
+508,16.92307270,NA,NA
+509,0.00000000,NA,NA
+510,0.00000000,NA,NA
+511,11.13817408,NA,NA
+512,46.18495205,NA,NA
+513,0.00000000,NA,NA
+514,29.04986504,NA,NA
+515,0.00000000,NA,NA
+516,0.00000000,NA,NA
+517,14.20797575,NA,NA
+518,0.00000000,NA,NA
+519,51.96424860,NA,NA
+520,0.00000000,NA,NA
+521,0.00000000,NA,NA
+522,0.00000000,NA,NA
+523,64.57454268,NA,NA
+524,13.42848302,NA,NA
+525,0.00000000,NA,NA
+526,0.00000000,NA,NA
+527,0.00000000,NA,NA
+528,14.82042815,NA,NA
+529,27.91406915,NA,NA
+530,0.00000000,NA,NA
+531,0.00000000,NA,NA
+532,0.00000000,NA,NA
+533,49.84702195,NA,NA
+534,56.96510248,NA,NA
+535,18.69821017,NA,NA
+536,0.00000000,NA,NA
+537,0.00000000,NA,NA
+538,0.00000000,NA,NA
+539,18.49816600,NA,NA
+540,0.00000000,NA,NA
+541,0.00000000,NA,NA
+542,0.00000000,NA,NA
+543,71.32667618,NA,NA
+544,0.00000000,NA,NA
+545,0.00000000,NA,NA
+546,97.38135293,NA,NA
+547,0.00000000,NA,NA
+548,23.14847806,NA,NA
+549,0.00000000,NA,NA
+550,0.00000000,NA,NA
+551,0.00000000,NA,NA
+552,0.00000000,NA,NA
+553,0.00000000,NA,NA
+554,0.00000000,NA,NA
+555,0.00000000,NA,NA
+556,0.00000000,NA,NA
+557,0.00000000,NA,NA
+558,0.00000000,NA,NA
+559,0.00000000,NA,NA
+560,12.21364341,NA,NA
+561,24.89929418,NA,NA
+563,0.00000000,NA,NA
+564,0.00000000,NA,NA
+565,0.00000000,NA,NA
+566,0.00000000,NA,NA
+567,22.68186574,NA,NA
+569,0.00000000,NA,NA
+570,0.00000000,NA,NA
+571,111.33146197,NA,NA
+573,0.00000000,NA,NA
+574,0.00000000,NA,NA
+575,0.00000000,NA,NA
+576,0.00000000,NA,NA
+577,0.00000000,NA,NA
+581,0.00000000,NA,NA
+582,0.00000000,NA,NA
+583,0.00000000,NA,NA
+584,0.00000000,NA,NA
+588,0.00000000,NA,NA
+592,0.00000000,NA,NA
+596,0.00000000,NA,NA
+597,0.00000000,NA,NA
+598,0.00000000,NA,NA
+600,0.00000000,NA,NA
+602,21.58373858,NA,NA
+603,0.00000000,NA,NA
+604,0.00000000,NA,NA
+605,0.00000000,NA,NA
+606,0.00000000,NA,NA
+609,15.19038838,NA,NA
+610,0.00000000,NA,NA
+611,0.00000000,NA,NA
+612,0.00000000,NA,NA
+613,9.71062768,NA,NA
+616,19.37040475,NA,NA
+617,0.00000000,NA,NA
+618,12.20127302,NA,NA
+619,0.00000000,NA,NA
+622,0.00000000,NA,NA
+623,34.36222518,NA,NA
+624,0.00000000,NA,NA
+625,51.14566285,NA,NA
+626,0.00000000,NA,NA
+627,0.00000000,NA,NA
+628,0.00000000,NA,NA
+630,0.00000000,NA,NA
+631,0.00000000,NA,NA
+632,0.00000000,NA,NA
+633,0.00000000,NA,NA
+634,0.00000000,NA,NA
+635,0.00000000,NA,NA
+636,78.28237624,NA,NA
+637,0.00000000,NA,NA
+638,0.00000000,NA,NA
+639,0.00000000,NA,NA
+640,0.00000000,NA,NA
+641,17.25271414,NA,NA
+642,0.00000000,NA,NA
+643,0.00000000,NA,NA
+644,0.00000000,NA,NA
+647,0.00000000,NA,NA
+648,0.00000000,NA,NA
+649,0.00000000,NA,NA
+650,0.00000000,NA,NA
+651,0.00000000,NA,NA
+652,0.00000000,NA,NA
+656,0.00000000,NA,NA
+657,0.00000000,NA,NA
+658,0.00000000,NA,NA
+659,0.00000000,NA,NA
+661,0.00000000,NA,NA
+662,0.00000000,NA,NA
+669,48.20671037,NA,NA
+670,0.00000000,NA,NA
+671,0.00000000,NA,NA
+673,0.00000000,NA,NA
+675,0.00000000,NA,NA
+676,0.00000000,NA,NA
+678,0.00000000,NA,NA
+679,0.00000000,NA,NA
+682,78.93905905,NA,NA
+686,0.00000000,NA,NA
+687,0.00000000,NA,NA
+691,0.00000000,NA,NA
+692,0.00000000,NA,NA
+693,0.00000000,NA,NA
+698,0.00000000,NA,NA
+702,0.00000000,NA,NA
+705,0.00000000,NA,NA
+709,0.00000000,NA,NA
+710,0.00000000,NA,NA
+711,0.00000000,NA,NA
+716,187.26591760,NA,NA
+717,0.00000000,NA,NA
+718,0.00000000,NA,NA
+722,0.00000000,NA,NA
+724,344.24132360,NA,NA
+725,269.10656620,NA,NA
+734,0.00000000,NA,NA
+736,0.00000000,NA,NA
+737,0.00000000,NA,NA
+740,0.00000000,NA,NA
+743,0.00000000,NA,NA
+744,39.36387970,NA,NA
+745,0.00000000,NA,NA
+746,0.00000000,NA,NA
+747,0.00000000,NA,NA
+749,0.00000000,NA,NA
+751,19.09092434,NA,NA
+752,101.48163182,NA,NA
+753,34.25453506,NA,NA
+754,0.00000000,NA,NA
+755,0.00000000,NA,NA
+756,0.00000000,NA,NA
+757,0.00000000,NA,NA
+758,0.00000000,NA,NA
+759,0.00000000,NA,NA
+760,0.00000000,NA,NA
+762,0.00000000,NA,NA
+764,0.00000000,NA,NA
+765,49.01960784,NA,NA
+766,0.00000000,NA,NA
+767,0.00000000,NA,NA
+770,14.75961245,NA,NA
+771,0.00000000,NA,NA
+773,0.00000000,NA,NA
+789,45.15895954,NA,NA
+790,28.98127210,NA,NA
+798,0.00000000,NA,NA
+800,0.00000000,NA,NA
+801,0.00000000,NA,NA
+802,0.00000000,NA,NA
+803,19.71649290,NA,NA
+804,124.68827930,NA,NA
+807,10.17947827,NA,NA
+810,0.00000000,NA,NA
+811,70.88176921,NA,NA
+813,0.00000000,NA,NA
+819,15.74578885,NA,NA
+820,26.33557359,NA,NA
+821,0.00000000,NA,NA
+825,12.60281411,NA,NA
+828,0.00000000,NA,NA
+839,0.00000000,NA,NA
+855,0.00000000,NA,NA
+862,0.00000000,NA,NA
+866,0.00000000,NA,NA
+868,0.00000000,NA,NA
+881,0.00000000,NA,NA