Merge branch 'main' of github.com:cmu-delphi/covidcast-indicators into geo

Author: sgsmob

.github/ISSUE_TEMPLATE/feature_release.md

@@ -0,0 +1,30 @@
+---
+name: Feature release 
+about: Begin the finishing work for features ready to be included in a release
+title: 'Release NEW_THING'
+labels: 'release'
+assignees: 'benjaminysmith'
+---
+
+- [Link to issue]()
+- [Link to PR]()
+- Proposed release version: <!-- eg 1.12 -->
+
+<!-- Additional information about the feature: -->
+
+
+<!-- relevant for most work -->
+
+- [ ] API [documentation](https://github.com/cmu-delphi/delphi-epidata/tree/main/docs/api) and/or [changelog](https://github.com/cmu-delphi/delphi-epidata/blob/main/docs/api/covidcast_changelog.md)
+- [ ] API mailing list notification
+
+<!-- relevant for new signals -->
+
+- [ ] Statistical review (usually [correlations](https://github.com/cmu-delphi/covidcast/tree/main/docs/R-notebooks))
+- [ ] Signal / source name review (usually [Roni](https://docs.google.com/document/d/10hGd4Evce4lJ4VkWaQEKFQxvmw2P4xyYGtIAWF52Sf8/edit?usp=sharing))
+
+<!-- relevant for new map signals -->
+
+- [ ] Visual review
+- [ ] [Signal description pop-up text](https://docs.google.com/document/d/1kDqRg8EaI4WQXMaUUbbCGPlsUqEql8kgXCNt6AvMA9I/edit?usp=sharing) review
+- [ ] [Map release notes](https://docs.google.com/document/d/1BpxGgIma_Lkd2kxtwEo2DBdHQ3zk6dHRz-leUIRlOIA/edit?usp=sharing)

ansible/files/usafacts-params-prod.json

@@ -0,0 +1,12 @@
+{
+  "export_start_date": "latest",
+  "static_file_dir": "./static",
+  "export_dir": "/common/covidcast/receiving/usa-facts",
+  "cache_dir": "./cache",
+  "base_url": "https://usafactsstatic.blob.core.windows.net/public/data/covid-19/covid_{metric}_usafacts.csv",
+  "aws_credentials": {
+    "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
+    "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
+  },
+  "bucket_name": "delphi-covidcast-indicator-output"
+}

ansible/templates/usafacts-params-prod.json.j2

@@ -96,13 +96,21 @@ def geo_reindex(self, data):
         """
         geo_map = GeoMapper()
         if self.geo == "county":
-            data_frame = geo_map.county_to_megacounty(
-                data, Config.MIN_DEN, Config.MAX_BACKWARDS_PAD_LENGTH,
-                thr_col="den", mega_col=self.geo)
+            data_frame = geo_map.fips_to_megacounty(data,
+                                                    Config.MIN_DEN,
+                                                    Config.MAX_BACKWARDS_PAD_LENGTH,
+                                                    thr_col="den",
+                                                    mega_col=self.geo)
         elif self.geo == "state":
-            data_frame = geo_map.county_to_state(data, state_id_col=self.geo)
+            data_frame = geo_map.replace_geocode(data,
+                                                 from_code="fips",
+                                                 new_col=self.geo,
+                                                 new_code="state_id")
+            data_frame[self.geo] = data_frame[self.geo]
         elif self.geo == "msa":
-            data_frame = geo_map.county_to_msa(data, msa_col=self.geo)
+            data_frame = geo_map.replace_geocode(data,
+                                                 from_code="fips",
+                                                 new_code=self.geo)
         elif self.geo == "hrr":
             data_frame = data  # data is already adjusted in aggregation step above
         else:
@@ -119,7 +127,6 @@ def geo_reindex(self, data):
         assert (
                 len(multiindex) <= (GeoConstants.MAX_GEO[self.geo] * len(self.fit_dates))
         ), "more loc-date pairs than maximum number of geographies x number of dates"
-
         # fill dataframe with missing dates using 0
         data_frame = data_frame.reindex(multiindex, fill_value=0)
         data_frame.fillna(0, inplace=True)

claims_hosp/delphi_claims_hosp/update_indicator.py

@@ -15,7 +15,9 @@ local_indicator="usafacts"
 cd "${WORKSPACE}/${local_indicator}" || exit
 
 # Linter
-env/bin/pylint delphi_"${local_indicator}"
+#env/bin/pylint delphi_"${local_indicator}"
+echo "Skip linting because we have weird breakage :( \
+  TODO: https://github.com/cmu-delphi/covidcast-indicators/issues/333"
 
 # Unit tests and code coverage
 cd tests || exit && \

claims_hosp/static/02_20_uszips.csv

@@ -11,7 +11,12 @@
 
 import numpy as np
 import pandas as pd
-from delphi_utils import read_params, create_export_csv, GeoMapper
+from delphi_utils import (
+    read_params,
+    create_export_csv,
+    S3ArchiveDiffer,
+    GeoMapper
+)
 
 from .geo import geo_map
 from .pull import pull_usafacts_data
@@ -73,6 +78,13 @@ def run_module():
     export_dir = params["export_dir"]
     base_url = params["base_url"]
     static_file_dir = params["static_file_dir"]
+    cache_dir = params["cache_dir"]
+
+    arch_diff = S3ArchiveDiffer(
+        cache_dir, export_dir,
+        params["bucket_name"], "usafacts",
+        params["aws_credentials"])
+    arch_diff.update_cache()
 
     map_df = pd.read_csv(
         join(static_file_dir, "fips_prop_pop.csv"), dtype={"fips": int}
@@ -105,3 +117,19 @@ def run_module():
             geo_res=geo_res,
             sensor=sensor_name,
         )
+
+    # Diff exports, and make incremental versions
+    _, common_diffs, new_files = arch_diff.diff_exports()
+
+    # Archive changed and new files only
+    to_archive = [f for f, diff in common_diffs.items() if diff is not None]
+    to_archive += new_files
+    _, fails = arch_diff.archive_exports(to_archive)
+
+    # Filter existing exports to exclude those that failed to archive
+    succ_common_diffs = {f: diff for f, diff in common_diffs.items() if f not in fails}
+    arch_diff.filter_exports(succ_common_diffs)
+
+    # Report failures: someone should probably look at them
+    for exported_file in fails:
+        print(f"Failed to archive '{exported_file}'")

jenkins/usafacts-jenkins-test.sh

@@ -3,5 +3,10 @@
   "static_file_dir": "./static",
   "export_dir": "./receiving",
   "cache_dir": "./cache",
-  "base_url": "https://usafactsstatic.blob.core.windows.net/public/data/covid-19/covid_{metric}_usafacts.csv"
+  "base_url": "https://usafactsstatic.blob.core.windows.net/public/data/covid-19/covid_{metric}_usafacts.csv",
+  "aws_credentials": {
+    "aws_access_key_id": "",
+    "aws_secret_access_key": ""
+  },
+  "bucket_name": ""
 }

usafacts/delphi_usafacts/run.py

@@ -1,19 +1,29 @@
 # -*- coding: utf-8 -*-
 
+from boto3 import Session
+from moto import mock_s3
 import pytest
 
 from os import listdir, remove
 from os.path import join
 
+from delphi_utils import read_params
 from delphi_usafacts.run import run_module
 
 
 @pytest.fixture(scope="session")
 def run_as_module():
     # Clean receiving directory
-    for fname in listdir("../receiving"):
+    for fname in listdir("receiving"):
         if fname[0] == ".":
             continue
-        remove(join("../receiving", fname))
+        remove(join("receiving", fname))
 
-    run_module()
+    with mock_s3():
+        # Create the fake bucket we will be using
+        params = read_params()
+        aws_credentials = params["aws_credentials"]
+        s3_client = Session(**aws_credentials).client("s3")
+        s3_client.create_bucket(Bucket=params["bucket_name"])
+
+        run_module()

usafacts/params.json.template

@@ -3,5 +3,10 @@
   "static_file_dir": "../static",
   "export_dir": "./receiving",
   "cache_dir": "./cache",
-  "base_url": "./test_data/small_{metric}.csv"
+  "base_url": "./test_data/small_{metric}.csv",
+  "aws_credentials": {
+    "aws_access_key_id": "FAKE_TEST_ACCESS_KEY_ID",
+    "aws_secret_access_key": "FAKE_TEST_SECRET_ACCESS_KEY"
+  },
+  "bucket_name": "test-bucket"
 }