Merge pull request #526 from sgsmob/hotfix

Fix issues causing `google_health` tests to fail

Author: krivard

ansible/templates/google_health-params-prod.json.j2

@@ -11,5 +11,7 @@
     "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
     "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
   },
-  "bucket_name": "delphi-covidcast-indicator-output"
+  "bucket_name": "delphi-covidcast-indicator-output",
+  "test": false,
+  "test_data_dir": ""
 }

ansible/templates/google_health-params-test.json.j2

@@ -11,5 +11,7 @@
     "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
     "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
   },
-  "bucket_name": "delphi-covidcast-indicator-output"
+  "bucket_name": "delphi-covidcast-indicator-output",
+  "test": true,
+  "test_data_dir": "./test_data/{geo_res}_sample.csv"
 }

google_health/delphi_google_health/run.py

@@ -107,18 +107,20 @@ def run_module():
                        start_date=start_date, receiving_dir=export_dir)
             export_csv(df_msa, MSA, signal, smooth=False,
                        start_date=start_date, receiving_dir=export_dir)
-    # Diff exports, and make incremental versions
-    _, common_diffs, new_files = arch_diff.diff_exports()
 
-    # Archive changed and new files only
-    to_archive = [f for f, diff in common_diffs.items() if diff is not None]
-    to_archive += new_files
-    _, fails = arch_diff.archive_exports(to_archive)
+    if not params["test"]:
+        # Diff exports, and make incremental versions
+        _, common_diffs, new_files = arch_diff.diff_exports()
+
+        # Archive changed and new files only
+        to_archive = [f for f, diff in common_diffs.items() if diff is not None]
+        to_archive += new_files
+        _, fails = arch_diff.archive_exports(to_archive)
 
-    # Filter existing exports to exclude those that failed to archive
-    succ_common_diffs = {f: diff for f, diff in common_diffs.items() if f not in fails}
-    arch_diff.filter_exports(succ_common_diffs)
+        # Filter existing exports to exclude those that failed to archive
+        succ_common_diffs = {f: diff for f, diff in common_diffs.items() if f not in fails}
+        arch_diff.filter_exports(succ_common_diffs)
 
-    # Report failures: someone should probably look at them
-    for exported_file in fails:
-        print(f"Failed to archive '{exported_file}'")
+        # Report failures: someone should probably look at them
+        for exported_file in fails:
+            print(f"Failed to archive '{exported_file}'")

google_health/tests/test_run.py

@@ -12,7 +12,7 @@ class TestRunModule:
 
     def test_class(self, run_as_module, wip_signal=read_params()["wip_signal"]):
         """Tests output file existence."""
-        if wip_signal is True:
+        if wip_signal:
             assert exists(join("receiving", "20200419_hrr_wip_raw_search.csv"))
             assert exists(join("receiving", "20200419_msa_wip_raw_search.csv"))
             assert exists(join("receiving", "20200419_state_wip_raw_search.csv"))
@@ -35,7 +35,7 @@ def test_class(self, run_as_module, wip_signal=read_params()["wip_signal"]):
 
     def test_match_old_raw_output(self, run_as_module, wip_signal=read_params()["wip_signal"]):
         """Tests that raw output files don't change over time."""
-        if wip_signal is True:
+        if wip_signal:
             files = [
                 "20200419_hrr_wip_raw_search.csv",
                 "20200419_msa_wip_raw_search.csv",
@@ -60,7 +60,7 @@ def test_match_old_raw_output(self, run_as_module, wip_signal=read_params()["wip
 
     def test_match_old_smoothed_output(self, run_as_module, wip_signal=read_params()["wip_signal"]):
         """Tests that smooth output files don't change over time."""
-        if wip_signal is True:
+        if wip_signal:
 
             files = [
                 "20200419_hrr_wip_smoothed_search.csv",