diff --git a/ansible/files/jhu-params-prod.json b/ansible/templates/jhu-params-prod.json.j2 similarity index 54% rename from ansible/files/jhu-params-prod.json rename to ansible/templates/jhu-params-prod.json.j2 index daf7d7097..14cceb0a7 100644 --- a/ansible/files/jhu-params-prod.json +++ b/ansible/templates/jhu-params-prod.json.j2 @@ -3,5 +3,10 @@ "static_file_dir": "./static", "export_dir": "/common/covidcast/receiving/jhu-csse/", "cache_dir": "./cache", - "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv" + "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv", + "aws_credentials": { + "aws_access_key_id": "{{ delphi_aws_access_key_id }}", + "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}" + }, + "bucket_name": "delphi-covidcast-indicator-output" } diff --git a/ansible/vars.yaml b/ansible/vars.yaml index bff5e0739..55af65fcc 100644 --- a/ansible/vars.yaml +++ b/ansible/vars.yaml @@ -9,3 +9,6 @@ pyenv_python_path: "/home/{{ runtime_user }}/.pyenv/versions/{{ python_version } # Indicators variables. google_health_api_key: "{{ vault_google_health_api_key }}" +delphi_aws_access_key_id: "{{ vault_delphi_aws_access_key_id }}" +delphi_aws_secret_access_key: "{{ vault_delphi_aws_secret_access_key }}" + diff --git a/ansible/vault.yaml b/ansible/vault.yaml index f10267e92..2d07321a9 100644 --- a/ansible/vault.yaml +++ b/ansible/vault.yaml @@ -1,9 +1,16 @@ $ANSIBLE_VAULT;1.1;AES256 -35386139306136373731386533393535343530623035366532343334326430313839336334646363 -6630646235616664386630623565363262643965376563660a386639663436616635323630356430 -35613139666439363161613232346463366637623238623963623364366166356131346363656432 -3831636539306230660a656430623166663262316230333137613633333337623338393837643331 -34306335636231366135383163336435613130396265333434623361376531396138353163653265 -33363963363032326164323161323036326436616333303738373866623335623664316565653461 -63646536633161623937363062663031353734303539343332633365326333353537316336383461 -64353265303234346633 +66386163643862646634343162646465663762643034303563333833633661333932646164656462 +6166646131623132393238336263623562373065643633310a663232373237396361623462613333 +62373663383565623263306539636431623230633065626363666531366662363065343066363031 +3738616663336665340a326138333634306137363837396366303861663064326333613662656630 +62306331646637326637363766366237663037306665343761643263646663316535343561623137 +63313365653535393639626465343232396261643239303430383138633135346466323834336665 +33633064353034613836313265613466623961373565363835343430373138376336363966316365 +35663664396436313432376264316663326130306134326231303234393561643436623039613136 +63366638396262383762383336643930343661636461646162653734336334306239383132643435 +39333665643738643966356431333830646561353263353063326330643731616130396466343339 +39346437653063303336626663623835613938633834396430353634383366386237353862643766 +37393738353231666565303031393839306463373461393761653866653330646534393832303264 +30323038646166366465396235623731343539313633326539663966333437623733626131653437 +62326632656462383835656235373664366566343866383938343639613737623631616231616135 +633863383761366461363532353137323936 diff --git a/google_health/delphi_google_health/run.py b/google_health/delphi_google_health/run.py index c44504bbf..58f344263 100644 --- a/google_health/delphi_google_health/run.py +++ b/google_health/delphi_google_health/run.py @@ -24,7 +24,7 @@ def run_module(): testing purposes). """ - #  read parameters + # read parameters params = read_params() ght_key = params["ght_key"] start_date = params["start_date"] @@ -51,7 +51,7 @@ def run_module(): # setup class to handle API calls ght = GoogleHealthTrends(ght_key=ght_key) - #  read data frame version of the data + # read data frame version of the data df_state = get_counts_states( ght, start_date, end_date, static_dir=static_dir, cache_dir=cache_dir ) @@ -64,7 +64,7 @@ def run_module(): for signal in signal_names: if signal.endswith(SMOOTHED): - #  export each geographic region, with both smoothed and unsmoothed data + # export each geographic region, with both smoothed and unsmoothed data export_csv(df_state, STATE, signal, smooth=True, receiving_dir=export_dir) export_csv(df_dma, DMA, signal, smooth=True, receiving_dir=export_dir) export_csv(df_hrr, HRR, signal, smooth=True, receiving_dir=export_dir) diff --git a/jhu/params.json.template b/jhu/params.json.template index b32c6ddc0..ad01dbf8c 100644 --- a/jhu/params.json.template +++ b/jhu/params.json.template @@ -6,6 +6,7 @@ "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv", "aws_credentials": { "aws_access_key_id": "", - "aws_secret_access_key": "", - } + "aws_secret_access_key": "" + }, + "bucket_name": "" } diff --git a/jhu/tests/conftest.py b/jhu/tests/conftest.py index 8513f2312..0df030e40 100644 --- a/jhu/tests/conftest.py +++ b/jhu/tests/conftest.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- +from boto3 import Session +from moto import mock_s3 import pytest from os import listdir, remove from os.path import join +from delphi_utils import read_params from delphi_jhu.run import run_module @@ -14,4 +17,11 @@ def run_as_module(): for fname in listdir("receiving"): remove(join("receiving", fname)) - run_module() + with mock_s3(): + # Create the fake bucket we will be using + params = read_params() + aws_credentials = params["aws_credentials"] + s3_client = Session(**aws_credentials).client("s3") + s3_client.create_bucket(Bucket=params["bucket_name"]) + + run_module() diff --git a/jhu/tests/params.json.template b/jhu/tests/params.json.template index 65f7136fc..0c874f0d9 100644 --- a/jhu/tests/params.json.template +++ b/jhu/tests/params.json.template @@ -3,5 +3,10 @@ "static_file_dir": "../static", "export_dir": "./receiving", "cache_dir": "./cache", - "base_url": "test_data/small_{metric}.csv" + "base_url": "test_data/small_{metric}.csv", + "aws_credentials": { + "aws_access_key_id": "FAKE_TEST_ACCESS_KEY_ID", + "aws_secret_access_key": "FAKE_TEST_SECRET_ACCESS_KEY" + }, + "bucket_name": "test-bucket" }