From cc92195e8712bc92165a595671b59c18c2cbfb74 Mon Sep 17 00:00:00 2001 From: Vishakha <59063647+vishakha1812@users.noreply.github.com> Date: Fri, 7 Aug 2020 14:59:49 -0700 Subject: [PATCH 1/5] handling wip signal in python template --- .../delphi_NAME/handle_wip_signal.py | 63 +++++++++++++++++++ _template_python/delphi_NAME/run.py | 20 +++++- _template_python/params.json.template | 3 +- .../tests/test_handle_wip_signal.py | 22 +++++++ 4 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 _template_python/delphi_NAME/handle_wip_signal.py create mode 100644 _template_python/tests/test_handle_wip_signal.py diff --git a/_template_python/delphi_NAME/handle_wip_signal.py b/_template_python/delphi_NAME/handle_wip_signal.py new file mode 100644 index 000000000..150bfe3b0 --- /dev/null +++ b/_template_python/delphi_NAME/handle_wip_signal.py @@ -0,0 +1,63 @@ +""" +Handle signal names. + +Author: Vishakha +Created: 2020-08-07 +""" + +from delphi_epidata import Epidata + + +def add_prefix(signal_names, wip_signal, prefix: str): + """Adds prefix to signal if there is a WIP signal + Parameters + ---------- + signal_names: List[str] + Names of signals to be exported + prefix : 'wip_' + prefix for new/non public signals + wip_signal : List[str] or bool + Either takes a list of wip signals: [], OR + incorporated all signals in the registry: True OR + no signals: False + Returns + ------- + List of signal names + wip/non wip signals for further computation + """ + if wip_signal in ("", False): + return signal_names + elif wip_signal and isinstance(wip_signal, bool): + return [ + (prefix + signal) if public_signal(signal) + else signal + for signal in signal_names + ] + elif isinstance(wip_signal, list): + for signal in wip_signal: + if public_signal(signal): + signal_names.append(prefix + signal) + signal_names.remove(signal) + return signal_names + else: + raise ValueError("Supply True | False or '' or [] | list()") + + +def public_signal(signal_): + """Checks if the signal name is already public using Epidata + Parameters + ---------- + signal_ : str + Name of the signal + Returns + ------- + bool + True if the signal is not present + False if the signal is present + """ + epidata_df = Epidata.covidcast_meta() + for index in range(len(epidata_df['epidata'])): + if 'signal' in epidata_df['epidata'][index]: + if epidata_df['epidata'][index]['signal'] == signal_: + return False + return True diff --git a/_template_python/delphi_NAME/run.py b/_template_python/delphi_NAME/run.py index d137d8e77..10a10f963 100644 --- a/_template_python/delphi_NAME/run.py +++ b/_template_python/delphi_NAME/run.py @@ -4,11 +4,25 @@ This module should contain a function called `run_module`, that is executed when the module is run with `python -m MODULE_NAME`. """ -import numpy as np -import pandas as pd from delphi_utils import read_params +from .handle_wip_signal import add_prefix +# Sample signals +SIGNALS = ["full_time_work_prop", + "covidnet", + "part_time_work_prop", + "completely_home_prop" + ] -def run_module(): +def run_module(): + """ + Calls the method for handling the wip signals + Returns + ------- + prints the updated signal names + """ params = read_params() + wip_signal = params["wip_signal"] + signal_names = add_prefix(SIGNALS, wip_signal, prefix="wip_") + print(signal_names) diff --git a/_template_python/params.json.template b/_template_python/params.json.template index b6a842c04..0a771326a 100644 --- a/_template_python/params.json.template +++ b/_template_python/params.json.template @@ -1,5 +1,6 @@ { "static_file_dir": "./static", "export_dir": "./receiving", - "cache_dir": "./cache" + "cache_dir": "./cache", + "wip_signal": "" } diff --git a/_template_python/tests/test_handle_wip_signal.py b/_template_python/tests/test_handle_wip_signal.py new file mode 100644 index 000000000..59f8fda14 --- /dev/null +++ b/_template_python/tests/test_handle_wip_signal.py @@ -0,0 +1,22 @@ +import unittest +from delphi_NAME.handle_wip_signal import add_prefix +from delphi_NAME.run import SIGNALS +from delphi_utils import read_params + +wip_signal = read_params()["wip_signal"] + + +def test_handle_wip_signal(): + assert isinstance(wip_signal, (list, bool)) or wip_signal == "", "Supply True | False or "" or [] | list()" + if isinstance(wip_signal, list): + assert set(wip_signal).issubset(set(SIGNALS)), "signal in params don't belong in the registry" + updated_signal_names = add_prefix(SIGNALS, wip_signal, prefix='wip_') + assert (len(updated_signal_names) >= len(SIGNALS)) + + +class MyTestCase(unittest.TestCase): + pass + + +if __name__ == '__main__': + unittest.main() From fe3efab76066c818813db58347686bab4e49fc9e Mon Sep 17 00:00:00 2001 From: Vishakha <59063647+vishakha1812@users.noreply.github.com> Date: Mon, 10 Aug 2020 16:06:20 -0700 Subject: [PATCH 2/5] changes in compliance with #205 --- _template_python/delphi_NAME/constants.py | 14 +++++ .../delphi_NAME/handle_wip_signal.py | 51 +++++++++---------- _template_python/delphi_NAME/run.py | 7 +-- _template_python/setup.py | 3 +- _template_python/tests/params.json.template | 3 +- .../tests/test_handle_wip_signal.py | 15 ++++-- 6 files changed, 54 insertions(+), 39 deletions(-) create mode 100644 _template_python/delphi_NAME/constants.py diff --git a/_template_python/delphi_NAME/constants.py b/_template_python/delphi_NAME/constants.py new file mode 100644 index 000000000..6c15f4e7b --- /dev/null +++ b/_template_python/delphi_NAME/constants.py @@ -0,0 +1,14 @@ +""" +Registry for signal names +""" + + +FULL_TIME = "full_time_work_prop" +PART_TIME = "part_time_work_prop" +COVIDNET = "covidnet" + +SIGNALS = [ + FULL_TIME, + PART_TIME, + COVIDNET +] diff --git a/_template_python/delphi_NAME/handle_wip_signal.py b/_template_python/delphi_NAME/handle_wip_signal.py index 150bfe3b0..709c83de0 100644 --- a/_template_python/delphi_NAME/handle_wip_signal.py +++ b/_template_python/delphi_NAME/handle_wip_signal.py @@ -5,7 +5,7 @@ Created: 2020-08-07 """ -from delphi_epidata import Epidata +import covidcast def add_prefix(signal_names, wip_signal, prefix: str): @@ -17,34 +17,34 @@ def add_prefix(signal_names, wip_signal, prefix: str): prefix : 'wip_' prefix for new/non public signals wip_signal : List[str] or bool - Either takes a list of wip signals: [], OR - incorporated all signals in the registry: True OR - no signals: False + a list of wip signals: [], OR + all signals in the registry: True OR + only signals that have never been published: False Returns ------- List of signal names wip/non wip signals for further computation """ - if wip_signal in ("", False): - return signal_names - elif wip_signal and isinstance(wip_signal, bool): + + if wip_signal is True: + return [prefix + signal for signal in signal_names] + if isinstance(wip_signal, list): + make_wip = set(wip_signal) + return[ + prefix + signal if signal in make_wip else signal + for signal in signal_names + ] + if wip_signal in {False, ""}: return [ - (prefix + signal) if public_signal(signal) - else signal + signal if public_signal(signal) + else prefix + signal for signal in signal_names ] - elif isinstance(wip_signal, list): - for signal in wip_signal: - if public_signal(signal): - signal_names.append(prefix + signal) - signal_names.remove(signal) - return signal_names - else: - raise ValueError("Supply True | False or '' or [] | list()") + raise ValueError("Supply True | False or '' or [] | list()") def public_signal(signal_): - """Checks if the signal name is already public using Epidata + """Checks if the signal name is already public using COVIDcast Parameters ---------- signal_ : str @@ -52,12 +52,11 @@ def public_signal(signal_): Returns ------- bool - True if the signal is not present - False if the signal is present + True if the signal is present + False if the signal is not present """ - epidata_df = Epidata.covidcast_meta() - for index in range(len(epidata_df['epidata'])): - if 'signal' in epidata_df['epidata'][index]: - if epidata_df['epidata'][index]['signal'] == signal_: - return False - return True + epidata_df = covidcast.metadata() + for index in range(len(epidata_df)): + if epidata_df['signal'][index] == signal_: + return True + return False diff --git a/_template_python/delphi_NAME/run.py b/_template_python/delphi_NAME/run.py index 10a10f963..5e1ea755b 100644 --- a/_template_python/delphi_NAME/run.py +++ b/_template_python/delphi_NAME/run.py @@ -6,14 +6,9 @@ """ from delphi_utils import read_params from .handle_wip_signal import add_prefix +from .constants import SIGNALS # Sample signals -SIGNALS = ["full_time_work_prop", - "covidnet", - "part_time_work_prop", - "completely_home_prop" - ] - def run_module(): """ diff --git a/_template_python/setup.py b/_template_python/setup.py index 6c9ba6048..ee535f6be 100644 --- a/_template_python/setup.py +++ b/_template_python/setup.py @@ -7,7 +7,8 @@ "pytest", "pytest-cov", "pylint", - "delphi-utils" + "delphi-utils", + "covidcast" ] setup( diff --git a/_template_python/tests/params.json.template b/_template_python/tests/params.json.template index 8dd8982e0..3d55211cd 100644 --- a/_template_python/tests/params.json.template +++ b/_template_python/tests/params.json.template @@ -1,5 +1,6 @@ { "static_file_dir": "../static", "export_dir": "./receiving", - "cache_dir": "./cache" + "cache_dir": "./cache", + "wip_signal": "" } diff --git a/_template_python/tests/test_handle_wip_signal.py b/_template_python/tests/test_handle_wip_signal.py index 59f8fda14..44d28e5de 100644 --- a/_template_python/tests/test_handle_wip_signal.py +++ b/_template_python/tests/test_handle_wip_signal.py @@ -7,11 +7,16 @@ def test_handle_wip_signal(): - assert isinstance(wip_signal, (list, bool)) or wip_signal == "", "Supply True | False or "" or [] | list()" - if isinstance(wip_signal, list): - assert set(wip_signal).issubset(set(SIGNALS)), "signal in params don't belong in the registry" - updated_signal_names = add_prefix(SIGNALS, wip_signal, prefix='wip_') - assert (len(updated_signal_names) >= len(SIGNALS)) + signal_names = add_prefix(SIGNALS, True, prefix="wip_") + assert all(s.startswith("wip_") for s in signal_names) + # Test wip_signal = list + signal_names = add_prefix(SIGNALS, [SIGNALS[0]], prefix="wip_") + assert signal_names[0].startswith("wip_") + assert all(not s.startswith("wip_") for s in signal_names[1:]) + # Test wip_signal = False + signal_names = add_prefix(["xyzzy", SIGNALS[0]], False, prefix="wip_") + assert signal_names[0].startswith("wip_") + assert all(not s.startswith("wip_") for s in signal_names[1:]) class MyTestCase(unittest.TestCase): From 7d9a59a6eef9dac7b38892ebeb56e04bf5e08ac1 Mon Sep 17 00:00:00 2001 From: Vishakha Srivastava <59063647+vishakha1812@users.noreply.github.com> Date: Tue, 11 Aug 2020 09:15:29 -0700 Subject: [PATCH 3/5] default prefix value --- _template_python/delphi_NAME/handle_wip_signal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_template_python/delphi_NAME/handle_wip_signal.py b/_template_python/delphi_NAME/handle_wip_signal.py index 709c83de0..04d42b57c 100644 --- a/_template_python/delphi_NAME/handle_wip_signal.py +++ b/_template_python/delphi_NAME/handle_wip_signal.py @@ -8,7 +8,7 @@ import covidcast -def add_prefix(signal_names, wip_signal, prefix: str): +def add_prefix(signal_names, wip_signal, prefix="wip_"): """Adds prefix to signal if there is a WIP signal Parameters ---------- From 9e637c9ffc19534a035ca512c152d2889c702930 Mon Sep 17 00:00:00 2001 From: Vishakha Srivastava <59063647+vishakha1812@users.noreply.github.com> Date: Tue, 11 Aug 2020 09:16:31 -0700 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: krivard --- _template_python/delphi_NAME/constants.py | 19 +++++++++++-------- .../tests/test_handle_wip_signal.py | 6 +++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/_template_python/delphi_NAME/constants.py b/_template_python/delphi_NAME/constants.py index 6c15f4e7b..94b3e32d3 100644 --- a/_template_python/delphi_NAME/constants.py +++ b/_template_python/delphi_NAME/constants.py @@ -3,12 +3,15 @@ """ -FULL_TIME = "full_time_work_prop" -PART_TIME = "part_time_work_prop" -COVIDNET = "covidnet" +## example: +# FULL_TIME = "full_time_work_prop" +# PART_TIME = "part_time_work_prop" +# COVIDNET = "covidnet" +# +# SIGNALS = [ +# FULL_TIME, +# PART_TIME, +# COVIDNET +# ] -SIGNALS = [ - FULL_TIME, - PART_TIME, - COVIDNET -] +SIGNALS = [] diff --git a/_template_python/tests/test_handle_wip_signal.py b/_template_python/tests/test_handle_wip_signal.py index 44d28e5de..13b0111bf 100644 --- a/_template_python/tests/test_handle_wip_signal.py +++ b/_template_python/tests/test_handle_wip_signal.py @@ -3,17 +3,17 @@ from delphi_NAME.run import SIGNALS from delphi_utils import read_params -wip_signal = read_params()["wip_signal"] def test_handle_wip_signal(): + # Test wip_signal = True (all signals should receive prefix) signal_names = add_prefix(SIGNALS, True, prefix="wip_") assert all(s.startswith("wip_") for s in signal_names) - # Test wip_signal = list + # Test wip_signal = list (only listed signals should receive prefix) signal_names = add_prefix(SIGNALS, [SIGNALS[0]], prefix="wip_") assert signal_names[0].startswith("wip_") assert all(not s.startswith("wip_") for s in signal_names[1:]) - # Test wip_signal = False + # Test wip_signal = False (only unpublished signals should receive prefix) signal_names = add_prefix(["xyzzy", SIGNALS[0]], False, prefix="wip_") assert signal_names[0].startswith("wip_") assert all(not s.startswith("wip_") for s in signal_names[1:]) From c2643e50b87feafad77ccf997fe45b2565a89095 Mon Sep 17 00:00:00 2001 From: Vishakha Srivastava <59063647+vishakha1812@users.noreply.github.com> Date: Tue, 11 Aug 2020 09:24:31 -0700 Subject: [PATCH 5/5] Removed unnecessary comment --- _template_python/delphi_NAME/run.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/_template_python/delphi_NAME/run.py b/_template_python/delphi_NAME/run.py index 5e1ea755b..c5342aba8 100644 --- a/_template_python/delphi_NAME/run.py +++ b/_template_python/delphi_NAME/run.py @@ -8,8 +8,6 @@ from .handle_wip_signal import add_prefix from .constants import SIGNALS -# Sample signals - def run_module(): """ Calls the method for handling the wip signals