diff --git a/safegraph/delphi_safegraph/constants.py b/safegraph/delphi_safegraph/constants.py new file mode 100644 index 000000000..334ca1df1 --- /dev/null +++ b/safegraph/delphi_safegraph/constants.py @@ -0,0 +1,18 @@ + + +HOME_DWELL = 'median_home_dwell_time' +COMPLETELY_HOME = 'completely_home_prop' +FULL_TIME_WORK = 'full_time_work_prop' +PART_TIME_WORK = 'part_time_work_prop' + +SIGNALS = [ + HOME_DWELL, + COMPLETELY_HOME, + FULL_TIME_WORK, + PART_TIME_WORK +] + +GEO_RESOLUTIONS = [ + 'county', + 'state', +] diff --git a/safegraph/delphi_safegraph/process.py b/safegraph/delphi_safegraph/process.py index 3c0b4e7bd..be7ccd4e1 100644 --- a/safegraph/delphi_safegraph/process.py +++ b/safegraph/delphi_safegraph/process.py @@ -1,14 +1,13 @@ -from delphi_epidata import Epidata - +import covidcast import numpy as np import pandas as pd +from .constants import HOME_DWELL, COMPLETELY_HOME, FULL_TIME_WORK, PART_TIME_WORK from .geo import FIPS_TO_STATE # Magic number for modular arithmetic; CBG -> FIPS MOD = 10000000 - def add_prefix(signal_names, wip_signal, prefix: str): """Adds prefix to signal if there is a WIP signal Parameters @@ -47,7 +46,7 @@ def add_prefix(signal_names, wip_signal, prefix: str): # Check if the signal name is public def public_signal(signal_): - """Checks if the signal name is already public using Epidata + """Checks if the signal name is already public using COVIDcast Parameters ---------- signal_ : str @@ -58,10 +57,10 @@ def public_signal(signal_): True if the signal is not present False if the signal is present """ - epidata_df = Epidata.covidcast_meta() - for index in range(len(epidata_df['epidata'])): - if 'signal' in epidata_df['epidata'][index]: - if epidata_df['epidata'][index]['signal'] == signal_: + epidata_df = covidcast.meta() + for index in range(len(epidata_df)): + if 'signal' in epidata_df[index]: + if epidata_df[index]['signal'] == signal_: return False return True @@ -91,10 +90,6 @@ def construct_signals(cbg_df, signal_names): Dataframe with columns: timestamp, county_fips, and {each signal described above}. """ - prefix = 'wip_' - COMPLETELY_HOME = 'completely_home_prop' - FULL_TIME_WORK = 'full_time_work_prop' - PART_TIME_WORK = 'part_time_work_prop' # Preparation cbg_df['timestamp'] = cbg_df['date_range_start'].apply( @@ -104,13 +99,13 @@ def construct_signals(cbg_df, signal_names): # Transformation: create signal not available in raw data for signal in signal_names: - if signal in (FULL_TIME_WORK, prefix + FULL_TIME_WORK): + if signal.endswith(FULL_TIME_WORK): cbg_df[signal] = (cbg_df['full_time_work_behavior_devices'] / cbg_df['device_count']) - elif signal in (COMPLETELY_HOME, prefix + COMPLETELY_HOME): + elif signal.endswith(COMPLETELY_HOME): cbg_df[signal] = (cbg_df['completely_home_device_count'] / cbg_df['device_count']) - elif signal in (PART_TIME_WORK, prefix + PART_TIME_WORK): + elif signal.endswith(PART_TIME_WORK): cbg_df[signal] = (cbg_df['part_time_work_behavior_devices'] / cbg_df['device_count']) diff --git a/safegraph/delphi_safegraph/run.py b/safegraph/delphi_safegraph/run.py index ddd849f31..a1a59c378 100644 --- a/safegraph/delphi_safegraph/run.py +++ b/safegraph/delphi_safegraph/run.py @@ -9,21 +9,9 @@ from delphi_utils import read_params +from .constants import SIGNALS, GEO_RESOLUTIONS from .process import process, add_prefix -SIGNALS = [ - 'median_home_dwell_time', - 'completely_home_prop', - 'full_time_work_prop', - 'part_time_work_prop' -] - -GEO_RESOLUTIONS = [ - 'county', - 'state', -] - - def run_module(): params = read_params() diff --git a/safegraph/setup.py b/safegraph/setup.py index db8e46400..0d3ca1f2a 100644 --- a/safegraph/setup.py +++ b/safegraph/setup.py @@ -2,6 +2,7 @@ from setuptools import find_packages required = [ + "covidcast" "numpy", "pandas", "pytest", diff --git a/safegraph/tests/params.json.template b/safegraph/tests/params.json.template new file mode 100644 index 000000000..7e6096821 --- /dev/null +++ b/safegraph/tests/params.json.template @@ -0,0 +1,12 @@ +{ + "static_file_dir": "./static", + "raw_data_dir": "/mnt/data/safegraph/", + "export_dir": "./receiving", + "cache_dir": "./cache", + "n_core": "12", + "aws_access_key_id": "", + "aws_secret_access_key": "", + "aws_default_region": "", + "aws_endpoint": "", + "wip_signal" : "" +}