adding conditional to fail if api fails

Author: aysim319

_delphi_utils_python/delphi_utils/validator/datafetcher.py

@@ -118,15 +118,13 @@ def get_geo_signal_combos(data_source, api_key):
     source_signal_mappings = {i['source']:i['db_source'] for i in
         meta_response.json()}
 
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
+    response = Epidata.covidcast_meta()
 
-        response = Epidata.covidcast_meta()
-
-        if response["result"] != 1:
-            # Something failed in the API and we did not get real metadata
-            raise RuntimeError("Error when fetching metadata from the API", response["message"])
+    if response["result"] != 1:
+        # Something failed in the API and we did not get real metadata
+        raise RuntimeError("Error when fetching metadata from the API", response["message"])
 
+    else:
         meta = pd.DataFrame.from_dict(response["epidata"])
         # note: this will fail for signals with weekly data, but currently not supported for validation
         meta = meta[meta["time_type"] == "day"]
@@ -183,12 +181,32 @@ def fetch_api_reference(data_source, start_date, end_date, geo_type, signal_type
         time_type="day",
         geo_type=geo_type,
         time_values=Epidata.range(start_date.strftime("%Y%m%d"), end_date.strftime("%Y%m%d")),
-        geo_value="*"
+        geo_value="*",
     )
     if response["result"] != 1:
-        # Something failed in the API and we did not get real metadata
+        # Something failed in the API and we did not get real signal data
         raise RuntimeError("Error when fetching signal data from the API", response["message"])
 
+    if response["message"] not in {"success", "no results"}:
+        warnings.warn(
+            "Problem obtaining data",
+            RuntimeWarning,
+            message=response["message"],
+            data_source=data_source,
+            signal=signal,
+            time_value=params["time_values"],
+            geo_type=geo_type,
+        )
+        logger.info(f"Trying calling covidcast again")
+        response = Epidata.covidcast(
+            data_source,
+            signal_type,
+            time_type="day",
+            geo_type=geo_type,
+            time_values=Epidata.range(start_date.strftime("%Y%m%d"), end_date.strftime("%Y%m%d")),
+            geo_value="*",
+        )
+
     api_df = None
     if len(response["epidata"]) > 0:
         api_df = pd.DataFrame.from_dict(response["epidata"])
@@ -204,8 +222,6 @@ def fetch_api_reference(data_source, start_date, end_date, geo_type, signal_type
 
     if api_df is None:
         raise APIDataFetchError("Error: no API data was returned " + error_context)
-    if not isinstance(api_df, pd.DataFrame):
-        raise APIDataFetchError("Error: API return value was not a dataframe " + error_context)
 
     column_names = ["geo_id", "val",
                     "se", "sample_size", "time_value"]

sir_complainsalot/delphi_sir_complainsalot/check_source.py

@@ -30,9 +30,12 @@ def __str__(self):
     def to_md(self):
         """Markdown formatted form of complaint."""
         return "*{source}* `{signal}` ({geos}) {message}; last updated {updated}.".format(
-            source=self.data_source, signal=self.signal, geos=", ".join(self.geo_types),
-            message=self.message, updated=self.last_updated.strftime("%Y-%m-%d"))
-
+            source=self.data_source,
+            signal=self.signal,
+            geos=", ".join(self.geo_types),
+            message=self.message,
+            updated=self.last_updated.strftime("%Y-%m-%d"),
+        )
 
 
 def check_source(data_source, meta, params, grace, logger):  # pylint: disable=too-many-locals
@@ -76,7 +79,8 @@ def check_source(data_source, meta, params, grace, logger):  # pylint: disable=t
             start_day=start_date.strftime("%Y-%m-%d"),
             end_day=end_date.strftime("%Y-%m-%d"),
             geo_type=row["geo_type"],
-            time_type=row["time_type"])
+            time_type=row["time_type"],
+        )
 
         response = Epidata.covidcast(
             data_source,
@@ -91,7 +95,7 @@ def check_source(data_source, meta, params, grace, logger):  # pylint: disable=t
         lag_calculated_from_api = False
         latest_data = None
 
-        if response["result"] == 1:
+        if response["result"] == 1 and response["message"] == "success":
             latest_data = pd.DataFrame.from_dict(response["epidata"])
             latest_data["time_value"] = _parse_datetimes(latest_data, "time_value")
             latest_data["issue"] = _parse_datetimes(latest_data, "issue")
@@ -101,6 +105,10 @@ def check_source(data_source, meta, params, grace, logger):  # pylint: disable=t
             current_lag_in_days = (datetime.now().date() - max(unique_dates)).days
             lag_calculated_from_api = True
 
+        else:
+            # Something failed in the API and we did not get real signal data
+            raise RuntimeError("Error when fetching signal data from the API", message=response["message"])
+
         logger.info("Signal lag",
                     current_lag_in_days = current_lag_in_days,
                     data_source = data_source,

sir_complainsalot/delphi_sir_complainsalot/date_utils.py

@@ -1,7 +1,8 @@
 from datetime import datetime
 
-from epiweeks import Week
 import pandas as pd
+from epiweeks import Week
+
 
 def _date_to_api_string(d: datetime, time_type: str = "day") -> str:
     """Convert a date object to a YYYYMMDD or YYYYMM string expected by the API."""
@@ -11,6 +12,7 @@ def _date_to_api_string(d: datetime, time_type: str = "day") -> str:
         return Week.fromdate(d).cdcformat()
     raise ValueError(f"Unknown time_type: {time_type}")
 
+
 def _parse_datetimes(df: pd.DataFrame, col: str, date_format: str = "%Y%m%d") -> pd.Series:
     """Convert a DataFrame date or epiweek column into datetimes.
 
@@ -19,10 +21,12 @@ def _parse_datetimes(df: pd.DataFrame, col: str, date_format: str = "%Y%m%d") ->
     format and return the date of the first day of the week.
     """
     df[col] = df[col].astype("str")
+
     def parse_row(row):
         if row["time_type"] == "day":
             return pd.to_datetime(row[col], format=date_format)
         if row["time_type"] == "week":
             return pd.to_datetime(Week(int(row[col][:4]), int(row[col][-2:])).startdate())
         return row[col]
-    return df.apply(parse_row, axis=1)
+
+    return df.apply(parse_row, axis=1)

sir_complainsalot/delphi_sir_complainsalot/run.py

@@ -30,7 +30,15 @@ def run_module():
     start_time = time.time()
     params = read_params()
     Epidata.auth = ("epidata", params["api_credentials"])
-    meta = pd.DataFrame.from_dict(Epidata.covidcast_meta().get("epidata", dict()))
+    response = Epidata.covidcast_meta()
+
+    meta = None
+    if response["result"] == 1:
+        meta = pd.DataFrame.from_dict(response["epidata"])
+    else:
+        # Something failed in the API and we did not get real metadata
+        raise RuntimeError("Error when fetching signal data from the API", response["message"])
+
     meta["max_time"] = _parse_datetimes(meta, "max_time")
     slack_notifier = None
     if "channel" in params and "slack_token" in params: