Merge pull request #1488 from cmu-delphi/ds/logger2

refactor: use delphi_utils.logger instead of copied file

Author: melange396

.git-blame-ignore-revs

@@ -20,3 +20,5 @@ b9ceb400d9248c8271e8342275664ac5524e335d
 07ed83e5768f717ab0f9a62a9209e4e2cffa058d
 # style(black): format wiki acquisition
 923852eafa86b8f8b182d499489249ba8f815843
+# lint: trailing whitespace changes
+1d5a71db152630698e8f525b007407a83e109a83

src/acquisition/covid_hosp/common/database.py

@@ -11,7 +11,7 @@
 
 # first party
 import delphi.operations.secrets as secrets
-from delphi.epidata.common.logger import get_structured_logger
+from delphi_utils import get_structured_logger
 
 Columndef = namedtuple("Columndef", "csv_name sql_name dtype")
 

src/acquisition/covidcast/csv_importer.py

@@ -13,10 +13,9 @@
 import pandas as pd
 
 # first party
-from delphi_utils import Nans
+from delphi_utils import get_structured_logger, Nans
 from delphi.utils.epiweek import delta_epiweeks
 from delphi.epidata.common.covidcast_row import CovidcastRow
-from delphi.epidata.common.logger import get_structured_logger
 
 DataFrameRow = NamedTuple('DFRow', [
   ('geo_id', str),

src/acquisition/covidcast/csv_to_database.py

@@ -11,7 +11,7 @@
 from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter, PathDetails
 from delphi.epidata.acquisition.covidcast.database import Database, DBLoadStateException
 from delphi.epidata.acquisition.covidcast.file_archiver import FileArchiver
-from delphi.epidata.common.logger import get_structured_logger
+from delphi_utils import get_structured_logger
 
 
 def get_argument_parser():

src/acquisition/covidcast/database.py

@@ -14,7 +14,7 @@
 
 # first party
 import delphi.operations.secrets as secrets
-from delphi.epidata.common.logger import get_structured_logger
+from delphi_utils import get_structured_logger
 from delphi.epidata.common.covidcast_row import CovidcastRow
 
 
@@ -117,28 +117,28 @@ def insert_or_update_batch(self, cc_rows: List[CovidcastRow], batch_size=2**20,
       get_structured_logger("insert_or_update_batch").fatal(err_msg)
       raise DBLoadStateException(err_msg)
 
-    # NOTE: `value_update_timestamp` is hardcoded to "NOW" (which is appropriate) and 
+    # NOTE: `value_update_timestamp` is hardcoded to "NOW" (which is appropriate) and
     #       `is_latest_issue` is hardcoded to 1 (which is temporary and addressed later in this method)
     insert_into_loader_sql = f'''
       INSERT INTO `{self.load_table}`
         (`source`, `signal`, `time_type`, `geo_type`, `time_value`, `geo_value`,
-        `value_updated_timestamp`, `value`, `stderr`, `sample_size`, `issue`, `lag`, 
+        `value_updated_timestamp`, `value`, `stderr`, `sample_size`, `issue`, `lag`,
         `is_latest_issue`, `missing_value`, `missing_stderr`, `missing_sample_size`)
       VALUES
-        (%s, %s, %s, %s, %s, %s, 
-        UNIX_TIMESTAMP(NOW()), %s, %s, %s, %s, %s, 
+        (%s, %s, %s, %s, %s, %s,
+        UNIX_TIMESTAMP(NOW()), %s, %s, %s, %s, %s,
         1, %s, %s, %s)
     '''
 
     # all load table entries are already marked "is_latest_issue".
     # if an entry in the load table is NOT in the latest table, it is clearly now the latest value for that key (so we do nothing (thanks to INNER join)).
     # if an entry *IS* in both load and latest tables, but latest table issue is newer, unmark is_latest_issue in load.
     fix_is_latest_issue_sql = f'''
-        UPDATE 
-            `{self.load_table}` JOIN `{self.latest_view}` 
-                USING (`source`, `signal`, `geo_type`, `geo_value`, `time_type`, `time_value`) 
-            SET `{self.load_table}`.`is_latest_issue`=0 
-            WHERE `{self.load_table}`.`issue` < `{self.latest_view}`.`issue` 
+        UPDATE
+            `{self.load_table}` JOIN `{self.latest_view}`
+                USING (`source`, `signal`, `geo_type`, `geo_value`, `time_type`, `time_value`)
+            SET `{self.load_table}`.`is_latest_issue`=0
+            WHERE `{self.load_table}`.`issue` < `{self.latest_view}`.`issue`
     '''
 
     # TODO: consider handling cc_rows as a generator instead of a list

src/acquisition/covidcast/file_archiver.py

@@ -6,7 +6,7 @@
 import shutil
 
 # first party
-from delphi.epidata.common.logger import get_structured_logger
+from delphi_utils import get_structured_logger
 
 class FileArchiver:
   """Archives files by moving and compressing."""

src/common/logger.py

@@ -7,7 +7,7 @@
 
 # first party
 from delphi.epidata.acquisition.covidcast.database import Database
-from delphi.epidata.common.logger import get_structured_logger
+from delphi_utils import get_structured_logger
 from delphi.epidata.client.delphi_epidata import Epidata
 
 def get_argument_parser():

src/maintenance/covidcast_meta_cache_updater.py

@@ -8,7 +8,7 @@
 
 # first party
 from delphi.epidata.acquisition.covidcast.database import Database
-from delphi.epidata.common.logger import get_structured_logger
+from delphi_utils import get_structured_logger
 
 
 def get_argument_parser():

src/maintenance/delete_batch.py

@@ -15,7 +15,7 @@
 # first party
 import covidcast
 import delphi.operations.secrets as secrets
-from delphi.epidata.common.logger import get_structured_logger
+from delphi_utils import get_structured_logger
 
 
 LOOKBACK_DAYS_FOR_COVERAGE = 56
@@ -150,11 +150,11 @@ def write_coverage(
 
     def get_enabled_signals(self) -> List[DashboardSignal]:
         """Retrieve all enabled signals from the database"""
-        select_statement = f'''SELECT `id`, 
+        select_statement = f'''SELECT `id`,
             `name`,
             `source`,
             `covidcast_signal`,
-            `latest_coverage_update`, 
+            `latest_coverage_update`,
             `latest_status_update`
             FROM `{Database.SIGNAL_TABLE_NAME}`
             WHERE `enabled`
@@ -208,7 +208,7 @@ def get_coverage(dashboard_signal: DashboardSignal) -> List[DashboardSignalCover
             lambda x: pd.to_datetime(Week(x // 100, x % 100).startdate()))
 
     signal_coverage_list = []
-    
+
     for _, row in count_by_geo_type_df.iterrows():
         signal_coverage = DashboardSignalCoverage(
             signal_id=dashboard_signal.db_id,