From 14ac2f4947492363cc5cf8715a1290fce537a72d Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Oct 2022 13:57:52 -0700 Subject: [PATCH 01/23] Server: add CovidcastRow helper class for testing --- src/acquisition/covidcast/covidcast_row.py | 270 ++++++++++++++++++ .../covidcast/test_covidcast_row.py | 97 +++++++ 2 files changed, 367 insertions(+) create mode 100644 src/acquisition/covidcast/covidcast_row.py create mode 100644 tests/acquisition/covidcast/test_covidcast_row.py diff --git a/src/acquisition/covidcast/covidcast_row.py b/src/acquisition/covidcast/covidcast_row.py new file mode 100644 index 000000000..10b59da95 --- /dev/null +++ b/src/acquisition/covidcast/covidcast_row.py @@ -0,0 +1,270 @@ +from dataclasses import asdict, dataclass, field, fields +from datetime import date +from typing import Any, ClassVar, Dict, Iterable, List, Optional + +import pandas as pd +from delphi_utils import Nans + +from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter +from delphi.epidata.server.utils.dates import day_to_time_value, time_value_to_day +from delphi.epidata.server.endpoints.covidcast_utils.model import PANDAS_DTYPES + + +@dataclass +class CovidcastRow: + """A container for the values of a single covidcast database row. + + Used for: + - inserting rows into the database + - creating test rows with default fields for testing + - converting from and to formats (dict, csv, df, kwargs) + - creating consistent views, with consistent data types (dict, csv, df) + + The rows are specified in 'v4_schema.sql'. The datatypes are made to match database. When writing to Pandas, the dtypes match the JIT model.py schema. + """ + + # Arguments. + source: str = "src" + signal: str = "sig" + time_type: str = "day" + geo_type: str = "county" + time_value: int = 20200202 + geo_value: str = "01234" + value: float = 10.0 + stderr: float = 10.0 + sample_size: float = 10.0 + missing_value: int = Nans.NOT_MISSING.value + missing_stderr: int = Nans.NOT_MISSING.value + missing_sample_size: int = Nans.NOT_MISSING.value + issue: Optional[int] = 20200202 + lag: Optional[int] = 0 + id: Optional[int] = None + direction: Optional[int] = None + direction_updated_timestamp: int = 0 + value_updated_timestamp: int = 20200202 + + # Classvars. + _api_row_ignore_fields: ClassVar = ["id", "direction_updated_timestamp", "value_updated_timestamp"] + _api_row_compatibility_ignore_fields: ClassVar = ["id", "direction_updated_timestamp", "value_updated_timestamp", "source"] + _db_row_ignore_fields: ClassVar = [] + _pandas_dtypes: ClassVar = PANDAS_DTYPES + + def __post_init__(self): + # Convert time values to ints by default. + self.time_value = day_to_time_value(self.time_value) if isinstance(self.time_value, date) else self.time_value + self.issue = day_to_time_value(self.issue) if isinstance(self.issue, date) else self.issue + self.value_updated_timestamp = day_to_time_value(self.value_updated_timestamp) if isinstance(self.value_updated_timestamp, date) else self.value_updated_timestamp + + def _sanity_check_fields(self, extra_checks: bool = True): + if self.issue and self.issue < self.time_value: + self.issue = self.time_value + + if self.issue: + self.lag = (time_value_to_day(self.issue) - time_value_to_day(self.time_value)).days + else: + self.lag = None + + # This sanity checking is already done in CsvImporter, but it's here so the testing class gets it too. + if pd.isna(self.value) and self.missing_value == Nans.NOT_MISSING: + self.missing_value = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + + if pd.isna(self.stderr) and self.missing_stderr == Nans.NOT_MISSING: + self.missing_stderr = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + + if pd.isna(self.sample_size) and self.missing_sample_size == Nans.NOT_MISSING: + self.missing_sample_size = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + + return self + + @staticmethod + def fromCsvRowValue(row_value: Optional[CsvImporter.RowValues], source: str, signal: str, time_type: str, geo_type: str, time_value: int, issue: int, lag: int): + """Create a CovidcastRow from a CsvImporter.RowValues object. + + Used in covidcast acquisition. + """ + if row_value is None: + return None + return CovidcastRow( + source, + signal, + time_type, + geo_type, + time_value, + row_value.geo_value, + row_value.value, + row_value.stderr, + row_value.sample_size, + row_value.missing_value, + row_value.missing_stderr, + row_value.missing_sample_size, + issue, + lag, + ) + + @staticmethod + def fromCsvRows(row_values: Iterable[Optional[CsvImporter.RowValues]], source: str, signal: str, time_type: str, geo_type: str, time_value: int, issue: int, lag: int): + """Create a generator of CovidcastRow from a list of CsvImporter.RowValues objects. + + Used in covidcast acquisition. + """ + return (CovidcastRow.fromCsvRowValue(row_value, source, signal, time_type, geo_type, time_value, issue, lag) for row_value in row_values) + + @staticmethod + def from_json(json: Dict[str, Any]) -> "CovidcastRow": + return CovidcastRow( + source=json["source"], + signal=json["signal"], + time_type=json["time_type"], + geo_type=json["geo_type"], + geo_value=json["geo_value"], + issue=json["issue"], + lag=json["lag"], + value=json["value"], + stderr=json["stderr"], + sample_size=json["sample_size"], + missing_value=json["missing_value"], + missing_stderr=json["missing_stderr"], + missing_sample_size=json["missing_sample_size"], + ) + + def as_dict(self, ignore_fields: Optional[List[str]] = None) -> dict: + d = asdict(self) + if ignore_fields: + for key in ignore_fields: + del d[key] + return d + + def as_dataframe(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFrame: + df = pd.DataFrame.from_records([self.as_dict(ignore_fields=ignore_fields)]) + # This is to mirror the types in model.py. + df = set_df_dtypes(df, self._pandas_dtypes) + return df + + @property + def api_row_df(self) -> pd.DataFrame: + """Returns a dataframe view into the row with the fields returned by the API server.""" + return self.as_dataframe(ignore_fields=self._api_row_ignore_fields) + + @property + def api_compatibility_row_df(self) -> pd.DataFrame: + """Returns a dataframe view into the row with the fields returned by the old API server (the PHP server).""" + return self.as_dataframe(ignore_fields=self._api_row_compatibility_ignore_fields) + + @property + def db_row_df(self) -> pd.DataFrame: + """Returns a dataframe view into the row with the fields returned by an all-field database query.""" + return self.as_dataframe(ignore_fields=self._db_row_ignore_fields) + + @property + def signal_pair(self): + return f"{self.source}:{self.signal}" + + @property + def geo_pair(self): + return f"{self.geo_type}:{self.geo_value}" + + @property + def time_pair(self): + return f"{self.time_type}:{self.time_value}" + + +@dataclass +class CovidcastRows: + # Arguments. + rows: List[CovidcastRow] = field(default_factory=list) + + # Classvars. + _api_row_ignore_fields: ClassVar = CovidcastRow._api_row_ignore_fields + _api_row_compatibility_ignore_fields: ClassVar = CovidcastRow._api_row_compatibility_ignore_fields + _db_row_ignore_fields: ClassVar = CovidcastRow._db_row_ignore_fields + _pandas_dtypes: ClassVar = CovidcastRow._pandas_dtypes + + @staticmethod + def from_args(sanity_check: bool = True, test_mode: bool = True, **kwargs: Dict[str, Iterable]): + """A convenience constructor. + + Handy for constructing batches of test cases. + + Example: + CovidcastRows.from_args(value=[1, 2, 3], time_value=[1, 2, 3]) will yield + CovidcastRows(rows=[CovidcastRow(value=1, time_value=1), CovidcastRow(value=2, time_value=2), CovidcastRow(value=3, time_value=3)]) + with all the defaults from CovidcastRow. + """ + # If any iterables were passed instead of lists, convert them to lists. + kwargs = {key: list(value) for key, value in kwargs.items()} + # All the arg values must be lists of the same length. + assert len(set(len(lst) for lst in kwargs.values())) == 1 + return CovidcastRows(rows=[CovidcastRow(**_kwargs) if not sanity_check else CovidcastRow(**_kwargs)._sanity_check_fields(extra_checks=test_mode) for _kwargs in transpose_dict(kwargs)]) + + @staticmethod + def from_records(records: Iterable[dict], sanity_check: bool = False): + """A convenience constructor. + + Default is different from from_args, because from_records is usually called on faux-API returns in tests, + where we don't want any values getting default filled in. + + You can use csv.DictReader before this to read a CSV file. + """ + records = list(records) + return CovidcastRows(rows=[CovidcastRow(**record) if not sanity_check else CovidcastRow(**record)._sanity_check_fields() for record in records]) + + def as_dicts(self, ignore_fields: Optional[List[str]] = None) -> List[dict]: + return [row.as_dict(ignore_fields=ignore_fields) for row in self.rows] + + def as_dataframe(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFrame: + if ignore_fields is None: + ignore_fields = [] + columns = [field.name for field in fields(CovidcastRow) if field.name not in ignore_fields] + if self.rows: + df = pd.concat([row.as_dataframe(ignore_fields=ignore_fields) for row in self.rows], ignore_index=True) + return df[columns] + else: + return pd.DataFrame(columns=columns) + + @property + def api_row_df(self) -> pd.DataFrame: + return self.as_dataframe(ignore_fields=self._api_row_ignore_fields) + + @property + def api_compatibility_row_df(self) -> pd.DataFrame: + return self.as_dataframe(ignore_fields=self._api_row_compatibility_ignore_fields) + + @property + def db_row_df(self) -> pd.DataFrame: + return self.as_dataframe(ignore_fields=self._db_row_ignore_fields) + + +def transpose_dict(d: Dict[Any, List[Any]]) -> List[Dict[Any, Any]]: + """Given a dictionary whose values are lists of the same length, turn it into a list of dictionaries whose values are the individual list entries. + + Example: + >>> transpose_dict(dict([["a", [2, 4, 6]], ["b", [3, 5, 7]], ["c", [10, 20, 30]]])) + [{"a": 2, "b": 3, "c": 10}, {"a": 4, "b": 5, "c": 20}, {"a": 6, "b": 7, "c": 30}] + """ + return [dict(zip(d.keys(), values)) for values in zip(*d.values())] + + +def check_valid_dtype(dtype): + try: + pd.api.types.pandas_dtype(dtype) + except TypeError: + raise ValueError(f"Invalid dtype {dtype}") + + +def set_df_dtypes(df: pd.DataFrame, dtypes: Dict[str, Any]) -> pd.DataFrame: + """Set the dataframe column datatypes.""" + [check_valid_dtype(d) for d in dtypes.values()] + + df = df.copy() + for k, v in dtypes.items(): + if k in df.columns: + df[k] = df[k].astype(v) + return df + + +def assert_frame_equal_no_order(df1: pd.DataFrame, df2: pd.DataFrame, index: List[str], **kwargs: Any) -> None: + """Assert that two DataFrames are equal, ignoring the order of rows.""" + # Remove any existing index. If it wasn't named, drop it. Set a new index and sort it. + df1 = df1.reset_index().drop(columns="index").set_index(index).sort_index() + df2 = df2.reset_index().drop(columns="index").set_index(index).sort_index() + pd.testing.assert_frame_equal(df1, df2, **kwargs) diff --git a/tests/acquisition/covidcast/test_covidcast_row.py b/tests/acquisition/covidcast/test_covidcast_row.py new file mode 100644 index 000000000..4700ec503 --- /dev/null +++ b/tests/acquisition/covidcast/test_covidcast_row.py @@ -0,0 +1,97 @@ +import unittest + +from pandas import DataFrame, date_range +from pandas.testing import assert_frame_equal + +from delphi_utils.nancodes import Nans +from delphi.epidata.server.utils.dates import day_to_time_value +from delphi.epidata.acquisition.covidcast.covidcast_row import set_df_dtypes, transpose_dict, CovidcastRow, CovidcastRows + +# py3tester coverage target (equivalent to `import *`) +__test_target__ = 'delphi.epidata.acquisition.covidcast.covidcast_row' + +class TestCovidcastRows(unittest.TestCase): + def test_transpose_dict(self): + assert transpose_dict(dict([["a", [2, 4, 6]], ["b", [3, 5, 7]], ["c", [10, 20, 30]]])) == [{"a": 2, "b": 3, "c": 10}, {"a": 4, "b": 5, "c": 20}, {"a": 6, "b": 7, "c": 30}] + + def test_CovidcastRow(self): + df = CovidcastRow(value=5.0).api_row_df + expected_df = set_df_dtypes(DataFrame.from_records([{ + "source": "src", + "signal": "sig", + "time_type": "day", + "geo_type": "county", + "time_value": 20200202, + "geo_value": "01234", + "value": 5.0, + "stderr": 10.0, + "sample_size": 10.0, + "missing_value": Nans.NOT_MISSING, + "missing_stderr": Nans.NOT_MISSING, + "missing_sample_size": Nans.NOT_MISSING, + "issue": 20200202, + "lag": 0, + "direction": None + }]), dtypes = CovidcastRow._pandas_dtypes) + assert_frame_equal(df, expected_df) + + df = CovidcastRow(value=5.0).api_compatibility_row_df + expected_df = set_df_dtypes(DataFrame.from_records([{ + "signal": "sig", + "time_type": "day", + "geo_type": "county", + "time_value": 20200202, + "geo_value": "01234", + "value": 5.0, + "stderr": 10.0, + "sample_size": 10.0, + "missing_value": Nans.NOT_MISSING, + "missing_stderr": Nans.NOT_MISSING, + "missing_sample_size": Nans.NOT_MISSING, + "issue": 20200202, + "lag": 0, + "direction": None + }]), dtypes = CovidcastRow._pandas_dtypes) + assert_frame_equal(df, expected_df) + + def test_CovidcastRows(self): + df = CovidcastRows.from_args(signal=["sig_base"] * 5 + ["sig_other"] * 5, time_value=date_range("2021-05-01", "2021-05-05").to_list() * 2, value=list(range(10))).api_row_df + expected_df = set_df_dtypes(DataFrame({ + "source": ["src"] * 10, + "signal": ["sig_base"] * 5 + ["sig_other"] * 5, + "time_type": ["day"] * 10, + "geo_type": ["county"] * 10, + "time_value": map(day_to_time_value, date_range("2021-05-01", "2021-05-5").to_list() * 2), + "geo_value": ["01234"] * 10, + "value": range(10), + "stderr": [10.0] * 10, + "sample_size": [10.0] * 10, + "missing_value": [Nans.NOT_MISSING] * 10, + "missing_stderr": [Nans.NOT_MISSING] * 10, + "missing_sample_size": [Nans.NOT_MISSING] * 10, + "issue": map(day_to_time_value, date_range("2021-05-01", "2021-05-5").to_list() * 2), + "lag": [0] * 10, + "direction": [None] * 10 + }), CovidcastRows._pandas_dtypes) + assert_frame_equal(df, expected_df) + + df = CovidcastRows.from_args( + signal=["sig_base"] * 5 + ["sig_other"] * 5, time_value=date_range("2021-05-01", "2021-05-05").to_list() * 2, value=list(range(10)) + ).api_compatibility_row_df + expected_df = set_df_dtypes(DataFrame({ + "signal": ["sig_base"] * 5 + ["sig_other"] * 5, + "time_type": ["day"] * 10, + "geo_type": ["county"] * 10, + "time_value": map(day_to_time_value, date_range("2021-05-01", "2021-05-5").to_list() * 2), + "geo_value": ["01234"] * 10, + "value": range(10), + "stderr": [10.0] * 10, + "sample_size": [10.0] * 10, + "missing_value": [Nans.NOT_MISSING] * 10, + "missing_stderr": [Nans.NOT_MISSING] * 10, + "missing_sample_size": [Nans.NOT_MISSING] * 10, + "issue": map(day_to_time_value, date_range("2021-05-01", "2021-05-5").to_list() * 2), + "lag": [0] * 10, + "direction": [None] * 10 + }), CovidcastRows._pandas_dtypes) + assert_frame_equal(df, expected_df) From 7124021662aefacf570ca325945332e1ff908351 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Oct 2022 15:33:05 -0700 Subject: [PATCH 02/23] Server: update csv_to_database to use CovidcastRow --- src/acquisition/covidcast/csv_to_database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/acquisition/covidcast/csv_to_database.py b/src/acquisition/covidcast/csv_to_database.py index 34cbad663..0abe53f1f 100644 --- a/src/acquisition/covidcast/csv_to_database.py +++ b/src/acquisition/covidcast/csv_to_database.py @@ -7,7 +7,8 @@ # first party from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter -from delphi.epidata.acquisition.covidcast.database import Database, CovidcastRow, DBLoadStateException +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow +from delphi.epidata.acquisition.covidcast.database import Database, DBLoadStateException from delphi.epidata.acquisition.covidcast.file_archiver import FileArchiver from delphi.epidata.acquisition.covidcast.logger import get_structured_logger From f1b361fe656577b340bc77e2fb2e04434d86dab4 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Oct 2022 15:31:46 -0700 Subject: [PATCH 03/23] Server: update test_db to use CovidcastRow --- integrations/acquisition/covidcast/test_db.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/integrations/acquisition/covidcast/test_db.py b/integrations/acquisition/covidcast/test_db.py index 3cd7e91a7..5daf8d272 100644 --- a/integrations/acquisition/covidcast/test_db.py +++ b/integrations/acquisition/covidcast/test_db.py @@ -1,10 +1,11 @@ -import unittest - from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.database import Database, CovidcastRow, DBLoadStateException + +from delphi.epidata.acquisition.covidcast.database import DBLoadStateException +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase import delphi.operations.secrets as secrets + # all the Nans we use here are just one value, so this is a shortcut to it: nmv = Nans.NOT_MISSING.value @@ -31,8 +32,8 @@ def _find_matches_for_row(self, row): def test_insert_or_update_with_nonempty_load_table(self): # make rows - a_row = self._make_placeholder_row()[0] - another_row = self._make_placeholder_row(time_value=self.DEFAULT_TIME_VALUE+1, issue=self.DEFAULT_ISSUE+1)[0] + a_row = CovidcastRow(time_value=20200202) + another_row = CovidcastRow(time_value=20200203, issue=20200203) # insert one self._db.insert_or_update_bulk([a_row]) # put something into the load table @@ -61,7 +62,7 @@ def test_id_sync(self): latest_view = 'epimetric_latest_v' # add a data point - base_row, _ = self._make_placeholder_row() + base_row = CovidcastRow() self._insert_rows([base_row]) # ensure the primary keys match in the latest and history tables matches = self._find_matches_for_row(base_row) @@ -71,7 +72,7 @@ def test_id_sync(self): old_pk_id = matches[latest_view][pk_column] # add a reissue for said data point - next_row, _ = self._make_placeholder_row() + next_row = CovidcastRow() next_row.issue += 1 self._insert_rows([next_row]) # ensure the new keys also match From 01a20113e4136d49957d0b08d421e78360db28ef Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Oct 2022 15:32:06 -0700 Subject: [PATCH 04/23] Server: update test_delete_batch to use CovidcastRow --- integrations/acquisition/covidcast/test_delete_batch.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/integrations/acquisition/covidcast/test_delete_batch.py b/integrations/acquisition/covidcast/test_delete_batch.py index 915c9341b..15ae7e2e2 100644 --- a/integrations/acquisition/covidcast/test_delete_batch.py +++ b/integrations/acquisition/covidcast/test_delete_batch.py @@ -5,13 +5,10 @@ import unittest from os import path -# third party -import mysql.connector - # first party -from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.database import Database, CovidcastRow import delphi.operations.secrets as secrets +from delphi.epidata.acquisition.covidcast.database import Database +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow # py3tester coverage target (equivalent to `import *`) __test_target__ = 'delphi.epidata.acquisition.covidcast.database' From 0ec5b288974d47ddfcc1eaa5aab90603fde46014 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Oct 2022 15:32:20 -0700 Subject: [PATCH 05/23] Server: update test_delphi_epidata to use CovidcastRow --- integrations/client/test_delphi_epidata.py | 110 ++++++++++----------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 625d2859d..cfeb83bd4 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -1,26 +1,28 @@ """Integration tests for delphi_epidata.py.""" # standard library -import unittest import time -from unittest.mock import patch, MagicMock from json import JSONDecodeError +from unittest.mock import MagicMock, patch -# third party -from aiohttp.client_exceptions import ClientResponseError -import mysql.connector +# first party import pytest +from aiohttp.client_exceptions import ClientResponseError -# first party -from delphi_utils import Nans -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.acquisition.covidcast.database import Database, CovidcastRow +# third party +import delphi.operations.secrets as secrets from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_covidcast_meta_cache +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase -import delphi.operations.secrets as secrets +from delphi.epidata.client.delphi_epidata import Epidata +from delphi_utils import Nans + # py3tester coverage target __test_target__ = 'delphi.epidata.client.delphi_epidata' +# all the Nans we use here are just one value, so this is a shortcut to it: +nmv = Nans.NOT_MISSING.value +IGNORE_FIELDS = ["id", "direction_updated_timestamp", "value_updated_timestamp", "source", "time_type", "geo_type"] def fake_epidata_endpoint(func): """This can be used as a decorator to enable a bogus Epidata endpoint to return 404 responses.""" @@ -30,9 +32,6 @@ def wrapper(*args): Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' return wrapper -# all the Nans we use here are just one value, so this is a shortcut to it: -nmv = Nans.NOT_MISSING.value - class DelphiEpidataPythonClientTests(CovidcastBase): """Tests the Python client.""" @@ -54,12 +53,12 @@ def test_covidcast(self): # insert placeholder data: three issues of one signal, one issue of another rows = [ - self._make_placeholder_row(issue=self.DEFAULT_ISSUE + i, value=i, lag=i)[0] + CovidcastRow(issue=20200202 + i, value=i, lag=i) for i in range(3) ] row_latest_issue = rows[-1] rows.append( - self._make_placeholder_row(signal="sig2")[0] + CovidcastRow(signal="sig2") ) self._insert_rows(rows) @@ -70,10 +69,11 @@ def test_covidcast(self): ) expected = [ - self.expected_from_row(row_latest_issue), - self.expected_from_row(rows[-1]) + row_latest_issue.as_dict(ignore_fields=IGNORE_FIELDS), + rows[-1].as_dict(ignore_fields=IGNORE_FIELDS) ] + self.assertEqual(response['epidata'], expected) # check result self.assertEqual(response, { 'result': 1, @@ -89,10 +89,10 @@ def test_covidcast(self): expected = [{ rows[0].signal: [ - self.expected_from_row(row_latest_issue, self.DEFAULT_MINUS + ['signal']), + row_latest_issue.as_dict(ignore_fields=IGNORE_FIELDS + ['signal']), ], rows[-1].signal: [ - self.expected_from_row(rows[-1], self.DEFAULT_MINUS + ['signal']), + rows[-1].as_dict(ignore_fields=IGNORE_FIELDS + ['signal']), ], }] @@ -109,12 +109,12 @@ def test_covidcast(self): **self.params_from_row(rows[0]) ) - expected = self.expected_from_row(row_latest_issue) + expected = [row_latest_issue.as_dict(ignore_fields=IGNORE_FIELDS)] # check result self.assertEqual(response_1, { 'result': 1, - 'epidata': [expected], + 'epidata': expected, 'message': 'success', }) @@ -124,13 +124,13 @@ def test_covidcast(self): **self.params_from_row(rows[0], as_of=rows[1].issue) ) - expected = self.expected_from_row(rows[1]) + expected = [rows[1].as_dict(ignore_fields=IGNORE_FIELDS)] # check result self.maxDiff=None self.assertEqual(response_1a, { 'result': 1, - 'epidata': [expected], + 'epidata': expected, 'message': 'success', }) @@ -141,8 +141,8 @@ def test_covidcast(self): ) expected = [ - self.expected_from_row(rows[0]), - self.expected_from_row(rows[1]) + rows[0].as_dict(ignore_fields=IGNORE_FIELDS), + rows[1].as_dict(ignore_fields=IGNORE_FIELDS) ] # check result @@ -158,12 +158,12 @@ def test_covidcast(self): **self.params_from_row(rows[0], lag=2) ) - expected = self.expected_from_row(row_latest_issue) + expected = [row_latest_issue.as_dict(ignore_fields=IGNORE_FIELDS)] # check result self.assertDictEqual(response_3, { 'result': 1, - 'epidata': [expected], + 'epidata': expected, 'message': 'success', }) with self.subTest(name='long request'): @@ -223,16 +223,16 @@ def test_geo_value(self): # insert placeholder data: three counties, three MSAs N = 3 rows = [ - self._make_placeholder_row(geo_type="county", geo_value=str(i)*5, value=i)[0] + CovidcastRow(geo_type="county", geo_value=str(i)*5, value=i) for i in range(N) ] + [ - self._make_placeholder_row(geo_type="msa", geo_value=str(i)*5, value=i*10)[0] + CovidcastRow(geo_type="msa", geo_value=str(i)*5, value=i*10) for i in range(N) ] self._insert_rows(rows) counties = [ - self.expected_from_row(rows[i]) for i in range(N) + rows[i].as_dict(ignore_fields=IGNORE_FIELDS) for i in range(N) ] def fetch(geo): @@ -241,31 +241,31 @@ def fetch(geo): ) # test fetch all - r = fetch('*') - self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], counties) + request = fetch('*') + self.assertEqual(request['message'], 'success') + self.assertEqual(request['epidata'], counties) # test fetch a specific region - r = fetch('11111') - self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [counties[1]]) + request = fetch('11111') + self.assertEqual(request['message'], 'success') + self.assertEqual(request['epidata'], [counties[1]]) # test fetch a specific yet not existing region - r = fetch('55555') - self.assertEqual(r['message'], 'no results') + request = fetch('55555') + self.assertEqual(request['message'], 'no results') # test fetch a multiple regions - r = fetch(['11111', '22222']) - self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [counties[1], counties[2]]) + request = fetch(['11111', '22222']) + self.assertEqual(request['message'], 'success') + self.assertEqual(request['epidata'], [counties[1], counties[2]]) # test fetch a multiple regions in another variant - r = fetch(['00000', '22222']) - self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [counties[0], counties[2]]) + request = fetch(['00000', '22222']) + self.assertEqual(request['message'], 'success') + self.assertEqual(request['epidata'], [counties[0], counties[2]]) # test fetch a multiple regions but one is not existing - r = fetch(['11111', '55555']) - self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [counties[1]]) + request = fetch(['11111', '55555']) + self.assertEqual(request['message'], 'success') + self.assertEqual(request['epidata'], [counties[1]]) # test fetch a multiple regions but specify no region - r = fetch([]) - self.assertEqual(r['message'], 'no results') + request = fetch([]) + self.assertEqual(request['message'], 'no results') def test_covidcast_meta(self): """Test that the covidcast_meta endpoint returns expected data.""" @@ -275,7 +275,7 @@ def test_covidcast_meta(self): # 2nd issue: 1 11 21 # 3rd issue: 2 12 22 rows = [ - self._make_placeholder_row(time_value=self.DEFAULT_TIME_VALUE + t, issue=self.DEFAULT_ISSUE + i, value=t*10 + i)[0] + CovidcastRow(time_value=2020_02_02 + t, issue=2020_02_02 + i, value=t*10 + i) for i in range(3) for t in range(3) ] self._insert_rows(rows) @@ -299,14 +299,14 @@ def test_covidcast_meta(self): signal=rows[0].signal, time_type=rows[0].time_type, geo_type=rows[0].geo_type, - min_time=self.DEFAULT_TIME_VALUE, - max_time=self.DEFAULT_TIME_VALUE + 2, + min_time=2020_02_02, + max_time=2020_02_02 + 2, num_locations=1, min_value=2., mean_value=12., max_value=22., stdev_value=8.1649658, # population stdev, not sample, which is 10. - max_issue=self.DEFAULT_ISSUE + 2, + max_issue=2020_02_02 + 2, min_lag=0, max_lag=0, # we didn't set lag when inputting data ) @@ -322,10 +322,10 @@ def test_async_epidata(self): # insert placeholder data: three counties, three MSAs N = 3 rows = [ - self._make_placeholder_row(geo_type="county", geo_value=str(i)*5, value=i)[0] + CovidcastRow(geo_type="county", geo_value=str(i)*5, value=i) for i in range(N) ] + [ - self._make_placeholder_row(geo_type="msa", geo_value=str(i)*5, value=i*10)[0] + CovidcastRow(geo_type="msa", geo_value=str(i)*5, value=i*10) for i in range(N) ] self._insert_rows(rows) From c49a64a4a5ab04d7495aecafa772f5f65e925e00 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Oct 2022 15:32:32 -0700 Subject: [PATCH 06/23] Server: update test_covidcast_endpoints to use CovidcastRow --- .../server/test_covidcast_endpoints.py | 110 ++++++++++++------ 1 file changed, 76 insertions(+), 34 deletions(-) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 54974a874..3208ceef2 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -1,7 +1,9 @@ """Integration tests for the custom `covidcast/*` endpoints.""" # standard library -from typing import Iterable, Dict, Any +from copy import copy +from itertools import accumulate, chain +from typing import Iterable, Dict, Any, List, Sequence import unittest from io import StringIO @@ -10,21 +12,22 @@ # third party import mysql.connector +from more_itertools import interleave_longest, windowed import requests import pandas as pd +import numpy as np from delphi_utils import Nans from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_cache - -from delphi.epidata.acquisition.covidcast.database import Database +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow, CovidcastRows, assert_frame_equal_no_order from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase # use the local instance of the Epidata API BASE_URL = "http://delphi_web_epidata/epidata/covidcast" +BASE_URL_OLD = "http://delphi_web_epidata/epidata/api.php" class CovidcastEndpointTests(CovidcastBase): - """Tests the `covidcast/*` endpoint.""" def localSetUp(self): @@ -32,19 +35,29 @@ def localSetUp(self): # reset the `covidcast_meta_cache` table (it should always have one row) self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - def _fetch(self, endpoint="/", **params): + def _fetch(self, endpoint="/", is_compatibility=False, **params): # make the request - response = requests.get( - f"{BASE_URL}{endpoint}", - params=params, - ) + if is_compatibility: + url = BASE_URL_OLD + params.setdefault("endpoint", "covidcast") + if params.get("source"): + params.setdefault("data_source", params.get("source")) + else: + url = f"{BASE_URL}{endpoint}" + response = requests.get(url, params=params) response.raise_for_status() return response.json() + def _diff_rows(self, rows: Sequence[float]): + return [float(x - y) if x is not None and y is not None else None for x, y in zip(rows[1:], rows[:-1])] + + def _smooth_rows(self, rows: Sequence[float]): + return [sum(e)/len(e) if None not in e else None for e in windowed(rows, 7)] + def test_basic(self): """Request a signal from the / endpoint.""" - rows = [self._make_placeholder_row(time_value=20200401 + i, value=i)[0] for i in range(10)] + rows = [CovidcastRow(time_value=20200401 + i, value=i) for i in range(10)] first = rows[0] self._insert_rows(rows) @@ -53,20 +66,52 @@ def test_basic(self): self.assertEqual(out["result"], -1) with self.subTest("simple"): - out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + out = self._fetch("/", signal=first.signal_pair, geo=first.geo_pair, time="day:*") + self.assertEqual(len(out["epidata"]), len(rows)) + + with self.subTest("unknown signal"): + rows = [CovidcastRow(source="jhu-csse", signal="confirmed_unknown", time_value=20200401 + i, value=i) for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + out = self._fetch("/", signal="jhu-csse:confirmed_unknown", geo=first.geo_pair, time="day:*") + out_values = [row["value"] for row in out["epidata"]] + expected_values = [float(row.value) for row in rows] + self.assertEqual(out_values, expected_values) + + def test_compatibility(self): + """Request at the /api.php endpoint.""" + rows = [CovidcastRow(source="src", signal="sig", time_value=20200401 + i, value=i) for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + with self.subTest("simple"): + # TODO: These tests aren't actually testing the compatibility endpoint. + out = self._fetch("/", signal=first.signal_pair, geo=first.geo_pair, time="day:*") self.assertEqual(len(out["epidata"]), len(rows)) + with self.subTest("unknown signal"): + rows = [CovidcastRow(source="jhu-csse", signal="confirmed_unknown", time_value=20200401 + i, value=i) for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + out = self._fetch("/", signal="jhu-csse:confirmed_unknown", geo=first.geo_pair, time="day:*") + out_values = [row["value"] for row in out["epidata"]] + expected_values = [float(row.value) for row in rows] + self.assertEqual(out_values, expected_values) + def test_trend(self): """Request a signal from the /trend endpoint.""" num_rows = 30 - rows = [self._make_placeholder_row(time_value=20200401 + i, value=i)[0] for i in range(num_rows)] + rows = [CovidcastRow(time_value=20200401 + i, value=i) for i in range(num_rows)] first = rows[0] last = rows[-1] ref = rows[num_rows // 2] self._insert_rows(rows) - out = self._fetch("/trend", signal=first.signal_pair(), geo=first.geo_pair(), date=last.time_value, window="20200401-20201212", basis=ref.time_value) + out = self._fetch("/trend", signal=first.signal_pair, geo=first.geo_pair, date=last.time_value, window="20200401-20201212", basis=ref.time_value) + self.assertEqual(out["result"], 1) self.assertEqual(len(out["epidata"]), 1) @@ -90,16 +135,17 @@ def test_trend(self): self.assertEqual(trend["max_value"], last.value) self.assertEqual(trend["max_trend"], "steady") + def test_trendseries(self): """Request a signal from the /trendseries endpoint.""" num_rows = 3 - rows = [self._make_placeholder_row(time_value=20200401 + i, value=num_rows - i)[0] for i in range(num_rows)] + rows = [CovidcastRow(time_value=20200401 + i, value=num_rows - i) for i in range(num_rows)] first = rows[0] last = rows[-1] self._insert_rows(rows) - out = self._fetch("/trendseries", signal=first.signal_pair(), geo=first.geo_pair(), date=last.time_value, window="20200401-20200410", basis=1) + out = self._fetch("/trendseries", signal=first.signal_pair, geo=first.geo_pair, date=last.time_value, window="20200401-20200410", basis=1) self.assertEqual(out["result"], 1) self.assertEqual(len(out["epidata"]), 3) @@ -127,6 +173,7 @@ def match_row(trend, row): self.assertEqual(trend["max_date"], first.time_value) self.assertEqual(trend["max_value"], first.value) self.assertEqual(trend["max_trend"], "steady") + with self.subTest("trend1"): trend = trends[1] match_row(trend, rows[1]) @@ -159,15 +206,15 @@ def test_correlation(self): """Request a signal from the /correlation endpoint.""" num_rows = 30 - reference_rows = [self._make_placeholder_row(signal="ref", time_value=20200401 + i, value=i)[0] for i in range(num_rows)] + reference_rows = [CovidcastRow(signal="ref", time_value=20200401 + i, value=i) for i in range(num_rows)] first = reference_rows[0] self._insert_rows(reference_rows) - other_rows = [self._make_placeholder_row(signal="other", time_value=20200401 + i, value=i)[0] for i in range(num_rows)] + other_rows = [CovidcastRow(signal="other", time_value=20200401 + i, value=i) for i in range(num_rows)] other = other_rows[0] self._insert_rows(other_rows) max_lag = 3 - out = self._fetch("/correlation", reference=first.signal_pair(), others=other.signal_pair(), geo=first.geo_pair(), window="20200401-20201212", lag=max_lag) + out = self._fetch("/correlation", reference=first.signal_pair, others=other.signal_pair, geo=first.geo_pair, window="20200401-20201212", lag=max_lag) self.assertEqual(out["result"], 1) df = pd.DataFrame(out["epidata"]) self.assertEqual(len(df), max_lag * 2 + 1) # -...0...+ @@ -185,31 +232,26 @@ def test_correlation(self): def test_csv(self): """Request a signal from the /csv endpoint.""" - rows = [self._make_placeholder_row(time_value=20200401 + i, value=i)[0] for i in range(10)] + rows = [CovidcastRow(time_value=20200401 + i, value=i) for i in range(10)] first = rows[0] self._insert_rows(rows) response = requests.get( f"{BASE_URL}/csv", - params=dict(signal=first.signal_pair(), start_day="2020-04-01", end_day="2020-12-12", geo_type=first.geo_type), + params=dict(signal=first.signal_pair, start_day="2020-04-01", end_day="2020-12-12", geo_type=first.geo_type), ) - response.raise_for_status() - out = response.text - df = pd.read_csv(StringIO(out), index_col=0) - self.assertEqual(df.shape, (len(rows), 10)) - self.assertEqual(list(df.columns), ["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "data_source"]) def test_backfill(self): """Request a signal from the /backfill endpoint.""" num_rows = 10 - issue_0 = [self._make_placeholder_row(time_value=20200401 + i, value=i, sample_size=1, lag=0, issue=20200401 + i)[0] for i in range(num_rows)] - issue_1 = [self._make_placeholder_row(time_value=20200401 + i, value=i + 1, sample_size=2, lag=1, issue=20200401 + i + 1)[0] for i in range(num_rows)] - last_issue = [self._make_placeholder_row(time_value=20200401 + i, value=i + 2, sample_size=3, lag=2, issue=20200401 + i + 2)[0] for i in range(num_rows)] # <-- the latest issues + issue_0 = [CovidcastRow(time_value=20200401 + i, value=i, sample_size=1, lag=0, issue=20200401 + i) for i in range(num_rows)] + issue_1 = [CovidcastRow(time_value=20200401 + i, value=i + 1, sample_size=2, lag=1, issue=20200401 + i + 1) for i in range(num_rows)] + last_issue = [CovidcastRow(time_value=20200401 + i, value=i + 2, sample_size=3, lag=2, issue=20200401 + i + 2) for i in range(num_rows)] # <-- the latest issues self._insert_rows([*issue_0, *issue_1, *last_issue]) first = issue_0[0] - out = self._fetch("/backfill", signal=first.signal_pair(), geo=first.geo_pair(), time="day:20200401-20201212", anchor_lag=3) + out = self._fetch("/backfill", signal=first.signal_pair, geo=first.geo_pair, time="day:20200401-20201212", anchor_lag=3) self.assertEqual(out["result"], 1) df = pd.DataFrame(out["epidata"]) self.assertEqual(len(df), 3 * num_rows) # num issues @@ -231,7 +273,7 @@ def test_meta(self): """Request a signal from the /meta endpoint.""" num_rows = 10 - rows = [self._make_placeholder_row(time_value=20200401 + i, value=i, source="fb-survey", signal="smoothed_cli")[0] for i in range(num_rows)] + rows = [CovidcastRow(time_value=20200401 + i, value=i, source="fb-survey", signal="smoothed_cli") for i in range(num_rows)] self._insert_rows(rows) first = rows[0] last = rows[-1] @@ -272,22 +314,22 @@ def test_coverage(self): num_geos_per_date = [10, 20, 30, 40, 44] dates = [20200401 + i for i in range(len(num_geos_per_date))] - rows = [self._make_placeholder_row(time_value=dates[i], value=i, geo_value=str(geo_value))[0] for i, num_geo in enumerate(num_geos_per_date) for geo_value in range(num_geo)] + rows = [CovidcastRow(time_value=dates[i], value=i, geo_value=str(geo_value)) for i, num_geo in enumerate(num_geos_per_date) for geo_value in range(num_geo)] self._insert_rows(rows) first = rows[0] with self.subTest("default"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type=first.geo_type, latest=dates[-1], format="json") + out = self._fetch("/coverage", signal=first.signal_pair, geo_type=first.geo_type, latest=dates[-1], format="json") self.assertEqual(len(out), len(num_geos_per_date)) self.assertEqual([o["time_value"] for o in out], dates) self.assertEqual([o["count"] for o in out], num_geos_per_date) with self.subTest("specify window"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type=first.geo_type, window=f"{dates[0]}-{dates[1]}", format="json") + out = self._fetch("/coverage", signal=first.signal_pair, geo_type=first.geo_type, window=f"{dates[0]}-{dates[1]}", format="json") self.assertEqual(len(out), 2) self.assertEqual([o["time_value"] for o in out], dates[:2]) self.assertEqual([o["count"] for o in out], num_geos_per_date[:2]) with self.subTest("invalid geo_type"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type="doesnt_exist", format="json") + out = self._fetch("/coverage", signal=first.signal_pair, geo_type="doesnt_exist", format="json") self.assertEqual(len(out), 0) From 48825fc26315dfd1175edb136e7a486aa4d2b313 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Oct 2022 15:32:40 -0700 Subject: [PATCH 07/23] Server: update test_covidcast to use CovidcastRow --- integrations/server/test_covidcast.py | 149 +++++++++++------------ tests/server/endpoints/test_covidcast.py | 4 - 2 files changed, 70 insertions(+), 83 deletions(-) diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 3de69f02c..047ceaec9 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -1,7 +1,7 @@ """Integration tests for the `covidcast` endpoint.""" # standard library -import json +from typing import Callable import unittest # third party @@ -10,12 +10,13 @@ # first party from delphi_utils import Nans +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase # use the local instance of the Epidata API +# TODO: should we still be using this? BASE_URL = 'http://delphi_web_epidata/epidata/api.php' - - +IGNORE_FIELDS = ["id", "direction_updated_timestamp", "value_updated_timestamp", "source", "time_type", "geo_type"] class CovidcastTests(CovidcastBase): """Tests the `covidcast` endpoint.""" @@ -24,28 +25,26 @@ def localSetUp(self): """Perform per-test setup.""" self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - def request_based_on_row(self, row, extract_response=lambda x: x.json(), **kwargs): + def request_based_on_row(self, row: CovidcastRow, extract_response: Callable = lambda x: x.json(), **kwargs): params = self.params_from_row(row, endpoint='covidcast', **kwargs) response = requests.get(BASE_URL, params=params) response.raise_for_status() response = extract_response(response) - expected = self.expected_from_row(row) - - return response, expected + return response def _insert_placeholder_set_one(self): - row, settings = self._make_placeholder_row() + row = CovidcastRow() self._insert_rows([row]) return row def _insert_placeholder_set_two(self): rows = [ - self._make_placeholder_row(geo_type='county', geo_value=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0] + CovidcastRow(geo_type='county', geo_value=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.) for i in [1, 2, 3] ] + [ # geo value intended to overlap with counties above - self._make_placeholder_row(geo_type='msa', geo_value=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0] + CovidcastRow(geo_type='msa', geo_value=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.) for i in [4, 5, 6] ] self._insert_rows(rows) @@ -53,11 +52,11 @@ def _insert_placeholder_set_two(self): def _insert_placeholder_set_three(self): rows = [ - self._make_placeholder_row(geo_type='county', geo_value='11111', time_value=2000_01_01+i, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=2-i)[0] + CovidcastRow(geo_type='county', geo_value='11111', time_value=2000_01_01+i, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=2-i) for i in [1, 2, 3] ] + [ # time value intended to overlap with 11111 above, with disjoint geo values - self._make_placeholder_row(geo_type='county', geo_value=str(i)*5, time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=5-i)[0] + CovidcastRow(geo_type='county', geo_value=str(i)*5, time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=5-i) for i in [4, 5, 6] ] self._insert_rows(rows) @@ -65,11 +64,11 @@ def _insert_placeholder_set_three(self): def _insert_placeholder_set_four(self): rows = [ - self._make_placeholder_row(source='src1', signal=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0] + CovidcastRow(source='src1', signal=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.) for i in [1, 2, 3] ] + [ # signal intended to overlap with the signal above - self._make_placeholder_row(source='src2', signal=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0] + CovidcastRow(source='src2', signal=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.) for i in [4, 5, 6] ] self._insert_rows(rows) @@ -82,10 +81,13 @@ def test_round_trip(self): row = self._insert_placeholder_set_one() # make the request - response, expected = self.request_based_on_row(row) + response = self.request_based_on_row(row) + + expected = [row.as_dict(ignore_fields=IGNORE_FIELDS)] + self.assertEqual(response, { 'result': 1, - 'epidata': [expected], + 'epidata': expected, 'message': 'success', }) @@ -142,32 +144,25 @@ def test_csv_format(self): # make the request # NB 'format' is a Python reserved word - response, _ = self.request_based_on_row( + response = self.request_based_on_row( row, extract_response=lambda resp: resp.text, **{'format':'csv'} ) - expected_response = ( - "geo_value,signal,time_value,direction,issue,lag,missing_value," + - "missing_stderr,missing_sample_size,value,stderr,sample_size\n" + - ",".join("" if x is None else str(x) for x in [ - row.geo_value, - row.signal, - row.time_value, - row.direction, - row.issue, - row.lag, - row.missing_value, - row.missing_stderr, - row.missing_sample_size, - row.value, - row.stderr, - row.sample_size - ]) + "\n" + + # TODO: This is a mess because of api.php. + column_order = [ + "geo_value", "signal", "time_value", "direction", "issue", "lag", "missing_value", + "missing_stderr", "missing_sample_size", "value", "stderr", "sample_size" + ] + expected = ( + row.api_compatibility_row_df + .assign(direction = None) + .to_csv(columns=column_order, index=False) ) # assert that the right data came back - self.assertEqual(response, expected_response) + self.assertEqual(response, expected) def test_raw_json_format(self): """Test generate raw json data.""" @@ -176,10 +171,12 @@ def test_raw_json_format(self): row = self._insert_placeholder_set_one() # make the request - response, expected = self.request_based_on_row(row, **{'format':'json'}) + response = self.request_based_on_row(row, **{'format':'json'}) + + expected = [row.as_dict(ignore_fields=IGNORE_FIELDS)] # assert that the right data came back - self.assertEqual(response, [expected]) + self.assertEqual(response, expected) def test_fields(self): """Test fields parameter""" @@ -188,7 +185,9 @@ def test_fields(self): row = self._insert_placeholder_set_one() # limit fields - response, expected = self.request_based_on_row(row, fields='time_value,geo_value') + response = self.request_based_on_row(row, fields='time_value,geo_value') + + expected = row.as_dict(ignore_fields=IGNORE_FIELDS) expected_all = { 'result': 1, 'epidata': [{ @@ -201,15 +200,14 @@ def test_fields(self): self.assertEqual(response, expected_all) # limit using invalid fields - response, _ = self.request_based_on_row(row, fields='time_value,geo_value,doesnt_exist') + response = self.request_based_on_row(row, fields='time_value,geo_value,doesnt_exist') # assert that the right data came back (only valid fields) self.assertEqual(response, expected_all) # limit exclude fields: exclude all except time_value and geo_value - - response, _ = self.request_based_on_row(row, fields=( + response = self.request_based_on_row(row, fields=( '-value,-stderr,-sample_size,-direction,-issue,-lag,-signal,' + '-missing_value,-missing_stderr,-missing_sample_size' )) @@ -222,18 +220,15 @@ def test_location_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected_counties = [ - self.expected_from_row(r) for r in rows[:3] - ] - + expected = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]] # make the request - response, _ = self.request_based_on_row(rows[0], geo_value="*") + response = self.request_based_on_row(rows[0], geo_value="*") self.maxDiff = None # assert that the right data came back self.assertEqual(response, { 'result': 1, - 'epidata': expected_counties, + 'epidata': expected, 'message': 'success', }) @@ -242,12 +237,10 @@ def test_signal_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_four() - expected_signals = [ - self.expected_from_row(r) for r in rows[:3] - ] + expected_signals = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]] # make the request - response, _ = self.request_based_on_row(rows[0], signals="*") + response = self.request_based_on_row(rows[0], signals="*") self.maxDiff = None # assert that the right data came back @@ -262,35 +255,33 @@ def test_geo_value(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected_counties = [ - self.expected_from_row(r) for r in rows[:3] - ] + expected = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]] def fetch(geo_value): # make the request - response, _ = self.request_based_on_row(rows[0], geo_value=geo_value) + response = self.request_based_on_row(rows[0], geo_value=geo_value) return response # test fetch a specific region r = fetch('11111') self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [expected_counties[0]]) + self.assertEqual(r['epidata'], expected[0:1]) # test fetch a specific yet not existing region r = fetch('55555') self.assertEqual(r['message'], 'no results') # test fetch multiple regions r = fetch('11111,22222') self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [expected_counties[0], expected_counties[1]]) + self.assertEqual(r['epidata'], expected[0:2]) # test fetch multiple noncontiguous regions r = fetch('11111,33333') self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [expected_counties[0], expected_counties[2]]) + self.assertEqual(r['epidata'], [expected[0], expected[2]]) # test fetch multiple regions but one is not existing r = fetch('11111,55555') self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [expected_counties[0]]) + self.assertEqual(r['epidata'], expected[0:1]) # test fetch empty region r = fetch('') self.assertEqual(r['message'], 'no results') @@ -300,12 +291,10 @@ def test_location_timeline(self): # insert placeholder data rows = self._insert_placeholder_set_three() - expected_timeseries = [ - self.expected_from_row(r) for r in rows[:3] - ] + expected_timeseries = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]] # make the request - response, _ = self.request_based_on_row(rows[0], time_values='20000101-20000105') + response = self.request_based_on_row(rows[0], time_values='20000101-20000105') # assert that the right data came back self.assertEqual(response, { @@ -331,15 +320,16 @@ def test_unique_key_constraint(self): def test_nullable_columns(self): """Missing values should be surfaced as null.""" - row, _ = self._make_placeholder_row( + row = CovidcastRow( stderr=None, sample_size=None, missing_stderr=Nans.OTHER.value, missing_sample_size=Nans.OTHER.value ) self._insert_rows([row]) # make the request - response, expected = self.request_based_on_row(row) - expected.update(stderr=None, sample_size=None) + response = self.request_based_on_row(row) + expected = row.as_dict(ignore_fields=IGNORE_FIELDS) + # expected.update(stderr=None, sample_size=None) # assert that the right data came back self.assertEqual(response, { @@ -353,18 +343,19 @@ def test_temporal_partitioning(self): # insert placeholder data rows = [ - self._make_placeholder_row(time_type=tt)[0] + CovidcastRow(time_type=tt) for tt in "hour day week month year".split() ] self._insert_rows(rows) # make the request - response, expected = self.request_based_on_row(rows[1], time_values="0-99999999") + response = self.request_based_on_row(rows[1], time_values="20000101-30010201") + expected = [rows[1].as_dict(ignore_fields=IGNORE_FIELDS)] # assert that the right data came back self.assertEqual(response, { 'result': 1, - 'epidata': [expected], + 'epidata': expected, 'message': 'success', }) @@ -375,37 +366,37 @@ def test_date_formats(self): rows = self._insert_placeholder_set_three() # make the request - response, expected = self.request_based_on_row(rows[0], time_values="20000102", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="20000102", geo_value="*") # assert that the right data came back self.assertEqual(len(response['epidata']), 2) # make the request - response, expected = self.request_based_on_row(rows[0], time_values="2000-01-02", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="2000-01-02", geo_value="*") # assert that the right data came back self.assertEqual(len(response['epidata']), 2) # make the request - response, expected = self.request_based_on_row(rows[0], time_values="20000102,20000103", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="20000102,20000103", geo_value="*") # assert that the right data came back - self.assertEqual(len(response['epidata']), 4) + self.assertEqual(len(response['epidata']), 2 * 2) # make the request - response, expected = self.request_based_on_row(rows[0], time_values="2000-01-02,2000-01-03", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="2000-01-02,2000-01-03", geo_value="*") # assert that the right data came back - self.assertEqual(len(response['epidata']), 4) + self.assertEqual(len(response['epidata']), 2 * 2) # make the request - response, expected = self.request_based_on_row(rows[0], time_values="20000102-20000104", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="20000102-20000104", geo_value="*") # assert that the right data came back - self.assertEqual(len(response['epidata']), 6) + self.assertEqual(len(response['epidata']), 2 * 3) # make the request - response, expected = self.request_based_on_row(rows[0], time_values="2000-01-02:2000-01-04", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="2000-01-02:2000-01-04", geo_value="*") # assert that the right data came back - self.assertEqual(len(response['epidata']), 6) + self.assertEqual(len(response['epidata']), 2 * 3) diff --git a/tests/server/endpoints/test_covidcast.py b/tests/server/endpoints/test_covidcast.py index b7ecdc263..823f9126a 100644 --- a/tests/server/endpoints/test_covidcast.py +++ b/tests/server/endpoints/test_covidcast.py @@ -5,10 +5,6 @@ from flask import Response from delphi.epidata.server.main import app -from delphi.epidata.server._params import ( - GeoPair, - TimePair, -) # py3tester coverage target __test_target__ = "delphi.epidata.server.endpoints.covidcast" From f6d7ae6702da0f28990a05aa2b2d988d5b3b2ee0 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Oct 2022 15:33:14 -0700 Subject: [PATCH 08/23] Server: update test_utils to use CovidcastRow --- src/acquisition/covidcast/test_utils.py | 44 ++++++------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/src/acquisition/covidcast/test_utils.py b/src/acquisition/covidcast/test_utils.py index 181dfac68..45f9fbfd0 100644 --- a/src/acquisition/covidcast/test_utils.py +++ b/src/acquisition/covidcast/test_utils.py @@ -1,7 +1,9 @@ +from typing import Sequence import unittest from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.database import Database, CovidcastRow +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow +from delphi.epidata.acquisition.covidcast.database import Database import delphi.operations.secrets as secrets # all the Nans we use here are just one value, so this is a shortcut to it: @@ -31,36 +33,20 @@ def tearDown(self): # close and destroy conenction to the database self._db.disconnect(False) del self._db + self.localTearDown() - DEFAULT_TIME_VALUE=2000_01_01 - DEFAULT_ISSUE=2000_01_01 - def _make_placeholder_row(self, **kwargs): - settings = { - 'source': 'src', - 'signal': 'sig', - 'geo_type': 'state', - 'geo_value': 'pa', - 'time_type': 'day', - 'time_value': self.DEFAULT_TIME_VALUE, - 'value': 0.0, - 'stderr': 1.0, - 'sample_size': 2.0, - 'missing_value': nmv, - 'missing_stderr': nmv, - 'missing_sample_size': nmv, - 'issue': self.DEFAULT_ISSUE, - 'lag': 0 - } - settings.update(kwargs) - return (CovidcastRow(**settings), settings) + def localTearDown(self): + # stub; override in subclasses to perform custom teardown. + # runs after database changes have been committed + pass - def _insert_rows(self, rows): + def _insert_rows(self, rows: Sequence[CovidcastRow]): # inserts rows into the database using the full acquisition process, including 'dbjobs' load into history & latest tables n = self._db.insert_or_update_bulk(rows) print(f"{n} rows added to load table & dispatched to v4 schema") self._db._connection.commit() # NOTE: this isnt expressly needed for our test cases, but would be if using external access (like through client lib) to ensure changes are visible outside of this db session - def params_from_row(self, row, **kwargs): + def params_from_row(self, row: CovidcastRow, **kwargs): ret = { 'data_source': row.source, 'signals': row.signal, @@ -71,13 +57,3 @@ def params_from_row(self, row, **kwargs): } ret.update(kwargs) return ret - - DEFAULT_MINUS=['time_type', 'geo_type', 'source'] - def expected_from_row(self, row, minus=DEFAULT_MINUS): - expected = dict(vars(row)) - # remove columns commonly excluded from output - # nb may need to add source or *_type back in for multiplexed queries - for key in ['id', 'direction_updated_timestamp'] + minus: - del expected[key] - return expected - From eac044f7bab01baa2f115ab3e2de538050a6ca32 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Oct 2022 15:50:17 -0700 Subject: [PATCH 09/23] Server: update TimePair to auto-sort tuples --- src/server/_params.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/server/_params.py b/src/server/_params.py index 2cef9725b..705556533 100644 --- a/src/server/_params.py +++ b/src/server/_params.py @@ -110,6 +110,10 @@ class TimePair: time_type: str time_values: Union[bool, TimeValues] + def __post_init__(self): + if isinstance(self.time_values, list): + self.time_values = [(min(time_value), max(time_value)) if isinstance(time_value, tuple) else time_value for time_value in self.time_values] + @property def is_week(self) -> bool: return self.time_type == 'week' From 5b998d580a30dc62e624e6a95c256622a632c6fe Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Oct 2022 16:02:09 -0700 Subject: [PATCH 10/23] Server: minor model.py data_source_by_id name update --- src/server/endpoints/covidcast_utils/model.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index 154bb3668..f84bc8974 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -202,7 +202,7 @@ def _load_data_sources(): data_sources, data_sources_df = _load_data_sources() -data_source_by_id = {d.source: d for d in data_sources} +data_sources_by_id = {d.source: d for d in data_sources} def _load_data_signals(sources: List[DataSource]): @@ -231,12 +231,11 @@ def _load_data_signals(sources: List[DataSource]): data_signals_by_key = {d.key: d for d in data_signals} # also add the resolved signal version to the signal lookup for d in data_signals: - source = data_source_by_id.get(d.source) + source = data_sources_by_id.get(d.source) if source and source.uses_db_alias: data_signals_by_key[(source.db_source, d.signal)] = d - def get_related_signals(signal: DataSignal) -> List[DataSignal]: return [s for s in data_signals if s != signal and s.signal_basename == signal.signal_basename] @@ -266,7 +265,7 @@ def create_source_signal_alias_mapper(source_signals: List[SourceSignalPair]) -> alias_to_data_sources: Dict[str, List[DataSource]] = {} transformed_pairs: List[SourceSignalPair] = [] for pair in source_signals: - source = data_source_by_id.get(pair.source) + source = data_sources_by_id.get(pair.source) if not source or not source.uses_db_alias: transformed_pairs.append(pair) continue From dc26e559feca082b2550aa496d6c95bd2623d881 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Wed, 30 Nov 2022 17:03:40 -0800 Subject: [PATCH 11/23] Server: update csv issue none handling --- src/server/endpoints/covidcast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/endpoints/covidcast.py b/src/server/endpoints/covidcast.py index 0c22e4573..2bd262e7e 100644 --- a/src/server/endpoints/covidcast.py +++ b/src/server/endpoints/covidcast.py @@ -401,7 +401,7 @@ def parse_row(i, row): "geo_value": row["geo_value"], "signal": row["signal"], "time_value": time_value_to_iso(row["time_value"]) if is_day else row["time_value"], - "issue": time_value_to_iso(row["issue"]) if is_day else row["issue"], + "issue": time_value_to_iso(row["issue"]) if is_day and row["issue"] is not None else row["issue"], "lag": row["lag"], "value": row["value"], "stderr": row["stderr"], From f8de0bae43c3f4af8290a38c2502cd6a8c1ddc1b Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 4 Nov 2022 16:57:57 -0700 Subject: [PATCH 12/23] Server: add type hints to _query --- src/server/_query.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/server/_query.py b/src/server/_query.py index 69607255f..2a5f7d7b8 100644 --- a/src/server/_query.py +++ b/src/server/_query.py @@ -9,8 +9,9 @@ Sequence, Tuple, Union, - cast + cast, ) +from flask import Response from sqlalchemy import text from sqlalchemy.engine import Row @@ -54,7 +55,7 @@ def filter_values( param_key: str, params: Dict[str, Any], formatter=lambda x: x, -): +) -> str: if not values: return "FALSE" # builds a SQL expression to filter strings (ex: locations) @@ -69,7 +70,7 @@ def filter_strings( values: Optional[Sequence[str]], param_key: str, params: Dict[str, Any], -): +) -> str: return filter_values(field, values, param_key, params) @@ -78,7 +79,7 @@ def filter_integers( values: Optional[Sequence[Union[Tuple[int, int], int]]], param_key: str, params: Dict[str, Any], -): +) -> str: return filter_values(field, values, param_key, params) @@ -87,7 +88,7 @@ def filter_dates( values: Optional[TimeValues], param_key: str, params: Dict[str, Any], -): +) -> str: ranges = time_values_to_ranges(values) return filter_values(field, ranges, param_key, params, date_string) @@ -199,7 +200,7 @@ def parse_row( fields_string: Optional[Sequence[str]] = None, fields_int: Optional[Sequence[str]] = None, fields_float: Optional[Sequence[str]] = None, -): +) -> Dict[str, Any]: keys = set(row.keys()) parsed = dict() if fields_string: @@ -235,7 +236,7 @@ def limit_query(query: str, limit: int) -> str: return full_query -def run_query(p: APrinter, query_tuple: Tuple[str, Dict[str, Any]]): +def run_query(p: APrinter, query_tuple: Tuple[str, Dict[str, Any]]) -> Iterable[Row]: query, params = query_tuple # limit rows + 1 for detecting whether we would have more full_query = text(limit_query(query, p.remaining_rows + 1)) @@ -255,7 +256,7 @@ def execute_queries( fields_int: Sequence[str], fields_float: Sequence[str], transform: Callable[[Dict[str, Any], Row], Dict[str, Any]] = _identity_transform, -): +) -> Response: """ execute the given queries and return the response to send them """ @@ -314,14 +315,14 @@ def execute_query( fields_int: Sequence[str], fields_float: Sequence[str], transform: Callable[[Dict[str, Any], Row], Dict[str, Any]] = _identity_transform, -): +) -> Response: """ execute the given query and return the response to send it """ return execute_queries([(query, params)], fields_string, fields_int, fields_float, transform) -def _join_l(value: Union[str, List[str]]): +def _join_l(value: Union[str, List[str]]) -> str: return ", ".join(value) if isinstance(value, (list, tuple)) else value From f2643253f697d346a93755593aee3ad19902253b Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Tue, 11 Oct 2022 14:46:48 -0700 Subject: [PATCH 13/23] Acquisition: update test_csv_uploading to remove Pandas warning --- integrations/acquisition/covidcast/test_csv_uploading.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index de3eb5f13..f975ecfa0 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -213,8 +213,8 @@ def test_uploading(self): "time_value": [20200419], "signal": [signal_name], "direction": [None]})], axis=1).rename(columns=uploader_column_rename) - expected_values_df["missing_value"].iloc[0] = Nans.OTHER - expected_values_df["missing_sample_size"].iloc[0] = Nans.NOT_MISSING + expected_values_df.loc[0, "missing_value"] = Nans.OTHER + expected_values_df.loc[0, "missing_sample_size"] = Nans.NOT_MISSING expected_values = expected_values_df.to_dict(orient="records") expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'} From 287af3312787e7a3134e24fa4f03ec536424dcb5 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Mon, 5 Dec 2022 12:31:48 -0800 Subject: [PATCH 14/23] Server: add PANDAS_DTYPES to model.py --- src/server/endpoints/covidcast_utils/model.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index f84bc8974..13a9b91d0 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -9,6 +9,28 @@ from ..._params import SourceSignalPair +PANDAS_DTYPES = { + "source": str, + "signal": str, + "time_type": str, + "time_value": "Int64", + "geo_type": str, + "geo_value": str, + "value": float, + "stderr": float, + "sample_size": float, + "missing_value": "Int8", + "missing_stderr": "Int8", + "missing_sample_size": "Int8", + "issue": "Int64", + "lag": "Int64", + "id": "Int64", + "direction": "Int8", + "direction_updated_timestamp": "Int64", + "value_updated_timestamp": "Int64", +} + + class HighValuesAre(str, Enum): bad = "bad" good = "good" From 0962cd37c368a766dda37242ad50ae1d1613034b Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Mon, 5 Dec 2022 12:39:55 -0800 Subject: [PATCH 15/23] Docker: add more_itertools==8.4.0 to Python and API images --- requirements.api.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.api.txt b/requirements.api.txt index d5cc0e63b..6ccafc1e1 100644 --- a/requirements.api.txt +++ b/requirements.api.txt @@ -2,6 +2,7 @@ epiweeks==2.1.2 Flask==2.2.2 itsdangerous<2.1 jinja2==3.0.3 +more_itertools==8.4.0 mysqlclient==2.1.1 newrelic orjson==3.4.7 From 9fc5535354a178c5c726d9867bc9650497cfaaf1 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Mon, 5 Dec 2022 13:11:40 -0800 Subject: [PATCH 16/23] Acquisition: update database.py to use CovidcastRow --- src/acquisition/covidcast/database.py | 53 ++------------------------- 1 file changed, 3 insertions(+), 50 deletions(-) diff --git a/src/acquisition/covidcast/database.py b/src/acquisition/covidcast/database.py index d21a27c35..92e45cafc 100644 --- a/src/acquisition/covidcast/database.py +++ b/src/acquisition/covidcast/database.py @@ -5,8 +5,8 @@ # third party import json +from typing import List import mysql.connector -import numpy as np from math import ceil from queue import Queue, Empty @@ -17,54 +17,7 @@ import delphi.operations.secrets as secrets from delphi.epidata.acquisition.covidcast.logger import get_structured_logger - -class CovidcastRow(): - """A container for all the values of a single covidcast row.""" - - @staticmethod - def fromCsvRowValue(row_value, source, signal, time_type, geo_type, time_value, issue, lag): - if row_value is None: return None - return CovidcastRow(source, signal, time_type, geo_type, time_value, - row_value.geo_value, - row_value.value, - row_value.stderr, - row_value.sample_size, - row_value.missing_value, - row_value.missing_stderr, - row_value.missing_sample_size, - issue, lag) - - @staticmethod - def fromCsvRows(row_values, source, signal, time_type, geo_type, time_value, issue, lag): - # NOTE: returns a generator, as row_values is expected to be a generator - return (CovidcastRow.fromCsvRowValue(row_value, source, signal, time_type, geo_type, time_value, issue, lag) - for row_value in row_values) - - def __init__(self, source, signal, time_type, geo_type, time_value, geo_value, value, stderr, - sample_size, missing_value, missing_stderr, missing_sample_size, issue, lag): - self.id = None - self.source = source - self.signal = signal - self.time_type = time_type - self.geo_type = geo_type - self.time_value = time_value - self.geo_value = geo_value # from CSV row - self.value = value # ... - self.stderr = stderr # ... - self.sample_size = sample_size # ... - self.missing_value = missing_value # ... - self.missing_stderr = missing_stderr # ... - self.missing_sample_size = missing_sample_size # from CSV row - self.direction_updated_timestamp = 0 - self.direction = None - self.issue = issue - self.lag = lag - - def signal_pair(self): - return f"{self.source}:{self.signal}" - - def geo_pair(self): - return f"{self.geo_type}:{self.geo_value}" +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow class DBLoadStateException(Exception): @@ -156,7 +109,7 @@ def do_analyze(self): def insert_or_update_bulk(self, cc_rows): return self.insert_or_update_batch(cc_rows) - def insert_or_update_batch(self, cc_rows, batch_size=2**20, commit_partial=False, suppress_jobs=False): + def insert_or_update_batch(self, cc_rows: List[CovidcastRow], batch_size=2**20, commit_partial=False, suppress_jobs=False): """ Insert new rows into the load table and dispatch into dimension and fact tables. """ From 0fa53d7ebca011c5cbce872e426d703dc1f51522 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Mon, 5 Dec 2022 13:25:10 -0800 Subject: [PATCH 17/23] Docker: bump API and Python pandas to 1.5.1 --- requirements.api.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.api.txt b/requirements.api.txt index 6ccafc1e1..7d20b1746 100644 --- a/requirements.api.txt +++ b/requirements.api.txt @@ -6,7 +6,7 @@ more_itertools==8.4.0 mysqlclient==2.1.1 newrelic orjson==3.4.7 -pandas==1.2.3 +pandas==1.5.1 python-dotenv==0.15.0 scipy==1.6.2 SQLAlchemy==1.4.40 From a1efe48884bcabfb2d0e80d24f7e1d2e92f3939a Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Mon, 5 Dec 2022 12:51:52 -0800 Subject: [PATCH 18/23] JIT: major feature commit * add smooth_diff * add model updates * add /trend endpoint * add /trendseries endpoint * add /csv endpoint * params with utility functions * update date utility functions --- deploy.json | 8 +- .../server/test_covidcast_endpoints.py | 325 +++++++++++++++--- requirements.api.txt | 1 + src/acquisition/covidcast/covidcast_row.py | 3 +- src/server/_config.py | 1 + src/server/_params.py | 33 +- src/server/_printer.py | 2 +- src/server/_validate.py | 21 ++ src/server/endpoints/covidcast.py | 275 ++++++++++++--- src/server/endpoints/covidcast_utils/model.py | 278 ++++++++++++++- .../endpoints/covidcast_utils/smooth_diff.py | 136 ++++++++ .../endpoints/covidcast_utils/test_utils.py | 176 ++++++++++ src/server/endpoints/covidcast_utils/trend.py | 4 + src/server/utils/__init__.py | 20 +- src/server/utils/dates.py | 51 +++ .../endpoints/covidcast_utils/test_model.py | 228 ++++++++++++ .../covidcast_utils/test_smooth_diff.py | 64 ++++ tests/server/test_params.py | 40 ++- tests/server/test_validate.py | 18 +- tests/server/utils/test_dates.py | 18 +- 20 files changed, 1585 insertions(+), 117 deletions(-) create mode 100644 src/server/endpoints/covidcast_utils/smooth_diff.py create mode 100644 src/server/endpoints/covidcast_utils/test_utils.py create mode 100644 tests/server/endpoints/covidcast_utils/test_model.py create mode 100644 tests/server/endpoints/covidcast_utils/test_smooth_diff.py diff --git a/deploy.json b/deploy.json index 45b45883e..b50bec4aa 100644 --- a/deploy.json +++ b/deploy.json @@ -32,6 +32,13 @@ "match": "^.*\\.(py)$", "add-header-comment": true }, + { + "type": "move", + "src": "src/server/utils", + "dst": "[[package]]/server/utils/", + "match": "^.*\\.(py)$", + "add-header-comment": true + }, { "type": "move", "src": "src/server/endpoints/covidcast_utils", @@ -39,7 +46,6 @@ "match": "^.*\\.(py)$", "add-header-comment": true }, - "// acquisition - fluview", { "type": "move", diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 3208ceef2..df923b6e4 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -1,32 +1,48 @@ """Integration tests for the custom `covidcast/*` endpoints.""" # standard library +import csv from copy import copy -from itertools import accumulate, chain -from typing import Iterable, Dict, Any, List, Sequence -import unittest from io import StringIO - -# from typing import Optional -from dataclasses import dataclass +from itertools import accumulate, chain +from typing import List +import numpy as np # third party -import mysql.connector -from more_itertools import interleave_longest, windowed -import requests import pandas as pd -import numpy as np -from delphi_utils import Nans +import pytest +import requests from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_cache from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow, CovidcastRows, assert_frame_equal_no_order from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase +from delphi.epidata.server.endpoints.covidcast_utils.test_utils import diff_df, reindex_df, diff_smooth_df +from delphi.epidata.server.utils.dates import iterate_over_range, iso_to_time_value # use the local instance of the Epidata API BASE_URL = "http://delphi_web_epidata/epidata/covidcast" BASE_URL_OLD = "http://delphi_web_epidata/epidata/api.php" +def _read_csv_str(txt: str) -> pd.DataFrame: + def gen(rows): + for row in rows: + row["value"] = float(row["value"]) if row["value"] else np.nan + row["stderr"] = float(row["stderr"]) if row["stderr"] else np.nan + row["sample_size"] = float(row["sample_size"]) if row["sample_size"] else np.nan + row["time_value"] = iso_to_time_value(row["time_value"]) + row["issue"] = iso_to_time_value(row["issue"]) if row["issue"] else np.nan + row["lag"] = int(row["lag"]) if row["lag"] else np.nan + row["geo_value"] = str(row["geo_value"]).zfill(5) + del row[""] + if "data_source" in row: + row["source"] = row["data_source"] + del row["data_source"] + yield row + + with StringIO(txt) as f: + return CovidcastRows.from_records(gen(csv.DictReader(f))).db_row_df + class CovidcastEndpointTests(CovidcastBase): """Tests the `covidcast/*` endpoint.""" @@ -48,12 +64,6 @@ def _fetch(self, endpoint="/", is_compatibility=False, **params): response.raise_for_status() return response.json() - def _diff_rows(self, rows: Sequence[float]): - return [float(x - y) if x is not None and y is not None else None for x, y in zip(rows[1:], rows[:-1])] - - def _smooth_rows(self, rows: Sequence[float]): - return [sum(e)/len(e) if None not in e else None for e in windowed(rows, 7)] - def test_basic(self): """Request a signal from the / endpoint.""" @@ -100,6 +110,101 @@ def test_compatibility(self): expected_values = [float(row.value) for row in rows] self.assertEqual(out_values, expected_values) + # JIT tests + def test_derived_signals(self): + # The base signal data. + data1 = CovidcastRows.from_args( + source = ["jhu-csse"] * 10, + signal = ["confirmed_cumulative_num"] * 10, + time_value = iterate_over_range(20200401, 20200410, inclusive=True), + geo_value = ["01"] * 10, + value = [i ** 2 for i in range(10)], + ) + data2 = CovidcastRows.from_args( + source = ["jhu-csse"] * 10, + signal = ["confirmed_cumulative_num"] * 10, + time_value = iterate_over_range(20200401, 20200410, inclusive=True), + geo_value = ["02"] * 10, + value = [2 * i ** 2 for i in range(10)], + ) + # A base signal with a time gap. + data3 = CovidcastRows.from_args( + source = ["jhu-csse"] * 15, + signal = ["confirmed_cumulative_num"] * 15, + time_value = chain(iterate_over_range(20200401, 20200410, inclusive=True), iterate_over_range(20200416, 20200420, inclusive=True)), + geo_value = ["03"] * 15, + value = [i ** 2 for i in chain(range(10), range(15, 20))], + ) + # Insert rows into database. + self._insert_rows(data1.rows + data2.rows + data3.rows) + # Fill the gap in data3. + data3_reindexed = reindex_df(data3.api_row_df) + data_df = pd.concat([data1.api_row_df, data2.api_row_df, data3_reindexed]) + # Get the expected derived signal values. + expected_diffed_df = diff_df(data_df, "confirmed_incidence_num").set_index(["signal", "geo_value", "time_value"]) + expected_smoothed_df = diff_smooth_df(data_df, "confirmed_7dav_incidence_num").set_index(["signal", "geo_value", "time_value"]) + expected_df = pd.concat([data_df.set_index(["signal", "geo_value", "time_value"]) , expected_diffed_df, expected_smoothed_df]) + + with self.subTest("diffed signal"): + out = self._fetch("/", signal="jhu-csse:confirmed_incidence_num", geo="county:01", time="day:20200401-20200410") + out_df = CovidcastRows.from_records(out["epidata"]).api_row_df.set_index(["signal", "geo_value", "time_value"]) + merged_df = pd.merge(out_df, expected_df, left_index=True, right_index=True, suffixes=["_out", "_expected"])[["value_out", "value_expected"]] + assert merged_df.empty is False + assert merged_df.value_out.to_numpy() == pytest.approx(merged_df.value_expected, nan_ok=True) + + with self.subTest("diffed signal, multiple geos"): + out = self._fetch("/", signal="jhu-csse:confirmed_incidence_num", geo="county:01,02", time="day:20200401-20200410") + out_df = CovidcastRows.from_records(out["epidata"]).api_row_df.set_index(["signal", "geo_value", "time_value"]) + merged_df = pd.merge(out_df, expected_df, left_index=True, right_index=True, suffixes=["_out", "_expected"])[["value_out", "value_expected"]] + assert merged_df.empty is False + assert merged_df.value_out.to_numpy() == pytest.approx(merged_df.value_expected, nan_ok=True) + + with self.subTest("smooth diffed signal"): + out = self._fetch("/", signal="jhu-csse:confirmed_7dav_incidence_num", geo="county:01,02", time="day:20200401-20200410") + out_df = CovidcastRows.from_records(out["epidata"]).api_row_df.set_index(["signal", "geo_value", "time_value"]) + assert merged_df.empty is False + assert merged_df.value_out.to_numpy() == pytest.approx(merged_df.value_expected, nan_ok=True) + + with self.subTest("diffed signal and smoothed signal in one request"): + out = self._fetch("/", signal="jhu-csse:confirmed_incidence_num;jhu-csse:confirmed_7dav_incidence_num", geo="county:01", time="day:20200401-20200410") + out_df = CovidcastRows.from_records(out["epidata"]).api_row_df.set_index(["signal", "geo_value", "time_value"]) + merged_df = pd.merge(out_df, expected_df, left_index=True, right_index=True, suffixes=["_out", "_expected"])[["value_out", "value_expected"]] + assert merged_df.empty is False + assert merged_df.value_out.to_numpy() == pytest.approx(merged_df.value_expected, nan_ok=True) + + with self.subTest("smoothing and diffing with a time gap and geo=*"): + # should fetch 7 extra day to make this work + out = self._fetch("/", signal="jhu-csse:confirmed_7dav_incidence_num", geo="county:*", time="day:20200407-20200420") + out_df = CovidcastRows.from_records(out["epidata"]).api_row_df.set_index(["signal", "geo_value", "time_value"]) + merged_df = pd.merge(out_df, expected_df, left_index=True, right_index=True, suffixes=["_out", "_expected"])[["value_out", "value_expected"]] + assert merged_df.empty is False + assert merged_df.value_out.to_numpy() == pytest.approx(merged_df.value_expected, nan_ok=True) + + with self.subTest("smoothing and diffing with a time gap and geo=* and time=*"): + out = self._fetch("/", signal="jhu-csse:confirmed_7dav_incidence_num", geo="county:*", time="day:*") + out_df = pd.DataFrame.from_records(out["epidata"]).set_index(["signal", "time_value", "geo_value"]) + merged_df = pd.merge(out_df, expected_df, left_index=True, right_index=True, suffixes=["_out", "_expected"])[["value_out", "value_expected"]] + assert merged_df.empty is False + assert merged_df.value_out.to_numpy() == pytest.approx(merged_df.value_expected, nan_ok=True) + + def test_compatibility(self): + """Request at the /api.php endpoint.""" + rows = [CovidcastRow(source="src", signal="sig", time_value=20200401 + i, value=i) for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + with self.subTest("simple"): + out = self._fetch(is_compatibility=True, source=first.source, signal=first.signal, geo=first.geo_pair, time="day:*") + self.assertEqual(len(out["epidata"]), len(rows)) + + def _diff_covidcast_rows(self, rows: List[CovidcastRow]) -> List[CovidcastRow]: + new_rows = list() + for x, y in zip(rows[1:], rows[:-1]): + new_row = copy(x) + new_row.value = x.value - y.value + new_rows.append(new_row) + return new_rows + def test_trend(self): """Request a signal from the /trend endpoint.""" @@ -110,30 +215,65 @@ def test_trend(self): ref = rows[num_rows // 2] self._insert_rows(rows) - out = self._fetch("/trend", signal=first.signal_pair, geo=first.geo_pair, date=last.time_value, window="20200401-20201212", basis=ref.time_value) + with self.subTest("no JIT"): + out = self._fetch("/trend", signal=first.signal_pair, geo=first.geo_pair, date=last.time_value, window="20200401-20201212", basis=ref.time_value) + self.assertEqual(out["result"], 1) + self.assertEqual(len(out["epidata"]), 1) + trend = out["epidata"][0] + self.assertEqual(trend["geo_type"], last.geo_type) + self.assertEqual(trend["geo_value"], last.geo_value) + self.assertEqual(trend["signal_source"], last.source) + self.assertEqual(trend["signal_signal"], last.signal) - self.assertEqual(out["result"], 1) - self.assertEqual(len(out["epidata"]), 1) - trend = out["epidata"][0] - self.assertEqual(trend["geo_type"], last.geo_type) - self.assertEqual(trend["geo_value"], last.geo_value) - self.assertEqual(trend["signal_source"], last.source) - self.assertEqual(trend["signal_signal"], last.signal) + self.assertEqual(trend["date"], last.time_value) + self.assertEqual(trend["value"], last.value) - self.assertEqual(trend["date"], last.time_value) - self.assertEqual(trend["value"], last.value) + self.assertEqual(trend["basis_date"], ref.time_value) + self.assertEqual(trend["basis_value"], ref.value) + self.assertEqual(trend["basis_trend"], "increasing") - self.assertEqual(trend["basis_date"], ref.time_value) - self.assertEqual(trend["basis_value"], ref.value) - self.assertEqual(trend["basis_trend"], "increasing") + self.assertEqual(trend["min_date"], first.time_value) + self.assertEqual(trend["min_value"], first.value) + self.assertEqual(trend["min_trend"], "increasing") + self.assertEqual(trend["max_date"], last.time_value) + self.assertEqual(trend["max_value"], last.value) + self.assertEqual(trend["max_trend"], "steady") - self.assertEqual(trend["min_date"], first.time_value) - self.assertEqual(trend["min_value"], first.value) - self.assertEqual(trend["min_trend"], "increasing") - self.assertEqual(trend["max_date"], last.time_value) - self.assertEqual(trend["max_value"], last.value) - self.assertEqual(trend["max_trend"], "steady") + num_rows = 30 + time_value_pairs = [(20200331, 0)] + [(20200401 + i, v) for i, v in enumerate(accumulate(range(num_rows)))] + rows = [CovidcastRow(source="jhu-csse", signal="confirmed_cumulative_num", time_value=t, value=v) for t, v in time_value_pairs] + self._insert_rows(rows) + diffed_rows = self._diff_covidcast_rows(rows) + for row in diffed_rows: + row.signal = "confirmed_incidence_num" + first = diffed_rows[0] + last = diffed_rows[-1] + ref = diffed_rows[num_rows // 2] + with self.subTest("use JIT"): + out = self._fetch("/trend", signal="jhu-csse:confirmed_incidence_num", geo=first.geo_pair, date=last.time_value, window="20200401-20201212", basis=ref.time_value) + + self.assertEqual(out["result"], 1) + self.assertEqual(len(out["epidata"]), 1) + trend = out["epidata"][0] + self.assertEqual(trend["geo_type"], last.geo_type) + self.assertEqual(trend["geo_value"], last.geo_value) + self.assertEqual(trend["signal_source"], last.source) + self.assertEqual(trend["signal_signal"], last.signal) + + self.assertEqual(trend["date"], last.time_value) + self.assertEqual(trend["value"], last.value) + + self.assertEqual(trend["basis_date"], ref.time_value) + self.assertEqual(trend["basis_value"], ref.value) + self.assertEqual(trend["basis_trend"], "increasing") + + self.assertEqual(trend["min_date"], first.time_value) + self.assertEqual(trend["min_value"], first.value) + self.assertEqual(trend["min_trend"], "increasing") + self.assertEqual(trend["max_date"], last.time_value) + self.assertEqual(trend["max_value"], last.value) + self.assertEqual(trend["max_trend"], "steady") def test_trendseries(self): @@ -202,6 +342,65 @@ def match_row(trend, row): self.assertEqual(trend["max_value"], first.value) self.assertEqual(trend["max_trend"], "decreasing") + num_rows = 3 + time_value_pairs = [(20200331, 0)] + [(20200401 + i, v) for i, v in enumerate(accumulate([num_rows - i for i in range(num_rows)]))] + rows = [CovidcastRow(source="jhu-csse", signal="confirmed_cumulative_num", time_value=t, value=v) for t, v in time_value_pairs] + self._insert_rows(rows) + diffed_rows = self._diff_covidcast_rows(rows) + for row in diffed_rows: + row.signal = "confirmed_incidence_num" + first = diffed_rows[0] + last = diffed_rows[-1] + + out = self._fetch("/trendseries", signal="jhu-csse:confirmed_incidence_num", geo=first.geo_pair, date=last.time_value, window="20200401-20200410", basis=1) + + self.assertEqual(out["result"], 1) + self.assertEqual(len(out["epidata"]), 3) + trends = out["epidata"] + + with self.subTest("trend0, JIT"): + trend = trends[0] + match_row(trend, first) + self.assertEqual(trend["basis_date"], None) + self.assertEqual(trend["basis_value"], None) + self.assertEqual(trend["basis_trend"], "unknown") + + self.assertEqual(trend["min_date"], last.time_value) + self.assertEqual(trend["min_value"], last.value) + self.assertEqual(trend["min_trend"], "increasing") + self.assertEqual(trend["max_date"], first.time_value) + self.assertEqual(trend["max_value"], first.value) + self.assertEqual(trend["max_trend"], "steady") + + with self.subTest("trend1"): + trend = trends[1] + match_row(trend, diffed_rows[1]) + self.assertEqual(trend["basis_date"], first.time_value) + self.assertEqual(trend["basis_value"], first.value) + self.assertEqual(trend["basis_trend"], "decreasing") + + self.assertEqual(trend["min_date"], last.time_value) + self.assertEqual(trend["min_value"], last.value) + self.assertEqual(trend["min_trend"], "increasing") + self.assertEqual(trend["max_date"], first.time_value) + self.assertEqual(trend["max_value"], first.value) + self.assertEqual(trend["max_trend"], "decreasing") + + with self.subTest("trend2"): + trend = trends[2] + match_row(trend, last) + self.assertEqual(trend["basis_date"], diffed_rows[1].time_value) + self.assertEqual(trend["basis_value"], diffed_rows[1].value) + self.assertEqual(trend["basis_trend"], "decreasing") + + self.assertEqual(trend["min_date"], last.time_value) + self.assertEqual(trend["min_value"], last.value) + self.assertEqual(trend["min_trend"], "steady") + self.assertEqual(trend["max_date"], first.time_value) + self.assertEqual(trend["max_value"], first.value) + self.assertEqual(trend["max_trend"], "decreasing") + + def test_correlation(self): """Request a signal from the /correlation endpoint.""" @@ -231,15 +430,53 @@ def test_correlation(self): def test_csv(self): """Request a signal from the /csv endpoint.""" - - rows = [CovidcastRow(time_value=20200401 + i, value=i) for i in range(10)] - first = rows[0] - self._insert_rows(rows) - - response = requests.get( - f"{BASE_URL}/csv", - params=dict(signal=first.signal_pair, start_day="2020-04-01", end_day="2020-12-12", geo_type=first.geo_type), + expected_columns = ["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "data_source"] + data = CovidcastRows.from_args( + time_value=pd.date_range("2020-04-01", "2020-04-10"), + value=range(10) + ) + self._insert_rows(data.rows) + first = data.rows[0] + with self.subTest("no JIT"): + response = requests.get( + f"{BASE_URL}/csv", + params=dict(signal=first.signal_pair, start_day="2020-04-01", end_day="2020-04-10", geo_type=first.geo_type), + ) + response.raise_for_status() + out = response.text + df = pd.read_csv(StringIO(out), index_col=0) + + self.assertEqual(df.shape, (len(data.rows), 10)) + self.assertEqual(list(df.columns), expected_columns) + + data = CovidcastRows.from_args( + source=["jhu-csse"] * 10, + signal=["confirmed_cumulative_num"] * 10, + time_value=pd.date_range("2020-04-01", "2020-04-10"), + value=accumulate(range(10)), ) + self._insert_rows(data.rows) + first = data.rows[0] + with self.subTest("use JIT"): + # Check that the data loaded correctly. + response = requests.get( + f"{BASE_URL}/csv", + params=dict(signal="jhu-csse:confirmed_cumulative_num", start_day="2020-04-01", end_day="2020-04-10", geo_type=first.geo_type), + ) + response.raise_for_status() + df = _read_csv_str(response.text) + expected_df = data.db_row_df + compare_cols = ["source", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "geo_value", "time_type"] + assert_frame_equal_no_order(df[compare_cols], expected_df[compare_cols], index=["source", "signal", "geo_value", "time_value"]) + + response = requests.get( + f"{BASE_URL}/csv", + params=dict(signal="jhu-csse:confirmed_incidence_num", start_day="2020-04-01", end_day="2020-04-10", geo_type=first.geo_type), + ) + response.raise_for_status() + df_diffed = _read_csv_str(response.text) + expected_df = diff_df(data.db_row_df, "confirmed_incidence_num", omit_left_boundary=True) + assert_frame_equal_no_order(df_diffed[compare_cols], expected_df[compare_cols], index=["source", "signal", "geo_value", "time_value"]) def test_backfill(self): """Request a signal from the /backfill endpoint.""" diff --git a/requirements.api.txt b/requirements.api.txt index 7d20b1746..171dcfe2b 100644 --- a/requirements.api.txt +++ b/requirements.api.txt @@ -1,3 +1,4 @@ +delphi_utils epiweeks==2.1.2 Flask==2.2.2 itsdangerous<2.1 diff --git a/src/acquisition/covidcast/covidcast_row.py b/src/acquisition/covidcast/covidcast_row.py index 10b59da95..918e24e99 100644 --- a/src/acquisition/covidcast/covidcast_row.py +++ b/src/acquisition/covidcast/covidcast_row.py @@ -219,7 +219,8 @@ def as_dataframe(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFram df = pd.concat([row.as_dataframe(ignore_fields=ignore_fields) for row in self.rows], ignore_index=True) return df[columns] else: - return pd.DataFrame(columns=columns) + df = pd.DataFrame(columns=columns) + return set_df_dtypes(df, self._pandas_dtypes) @property def api_row_df(self) -> pd.DataFrame: diff --git a/src/server/_config.py b/src/server/_config.py index 0474267f3..10193f7ab 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -8,6 +8,7 @@ MAX_RESULTS = int(10e6) MAX_COMPATIBILITY_RESULTS = int(3650) +MAX_SMOOTHER_WINDOW = 30 SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", "sqlite:///test.db") diff --git a/src/server/_params.py b/src/server/_params.py index 705556533..f4c4a310b 100644 --- a/src/server/_params.py +++ b/src/server/_params.py @@ -1,10 +1,11 @@ -from math import inf import re from dataclasses import dataclass +from itertools import groupby +from math import inf from typing import List, Optional, Sequence, Tuple, Union from flask import request - +from more_itertools import flatten from ._exceptions import ValidationFailedException from .utils import days_in_range, weeks_in_range, guess_time_value_is_day, guess_time_value_is_week, TimeValues, days_to_ranges, weeks_to_ranges @@ -92,9 +93,35 @@ def count(self) -> float: return inf if self.signal else 0 return len(self.signal) + def add_signal(self, signal: str) -> None: + if not isinstance(self.signal, bool): + self.signal.append(signal) + + def __hash__(self) -> int: + return hash((self.source, self.signal if self.signal is isinstance(self.signal, bool) else tuple(self.signal))) + + +def _combine_source_signal_pairs(source_signal_pairs: List[SourceSignalPair]) -> List[SourceSignalPair]: + """Combine SourceSignalPairs with the same source into a single SourceSignalPair object. + + Example: + [SourceSignalPair("src", ["sig1", "sig2"]), SourceSignalPair("src", ["sig2", "sig3"])] will be merged + into [SourceSignalPair("src", ["sig1", "sig2", "sig3])]. + """ + source_signal_pairs_grouped = groupby(sorted(source_signal_pairs, key=lambda x: x.source), lambda x: x.source) + source_signal_pairs_combined = [] + for source, group in source_signal_pairs_grouped: + group = list(group) + if any(x.signal == True for x in group): + combined_signals = True + else: + combined_signals = sorted(set(flatten(x.signal for x in group))) + source_signal_pairs_combined.append(SourceSignalPair(source, combined_signals)) + return source_signal_pairs_combined + def parse_source_signal_arg(key: str = "signal") -> List[SourceSignalPair]: - return [SourceSignalPair(source, signals) for [source, signals] in _parse_common_multi_arg(key)] + return _combine_source_signal_pairs(SourceSignalPair(source, signals) for [source, signals] in _parse_common_multi_arg(key)) def parse_single_source_signal_arg(key: str) -> SourceSignalPair: diff --git a/src/server/_printer.py b/src/server/_printer.py index bbe3ee10e..04196c71d 100644 --- a/src/server/_printer.py +++ b/src/server/_printer.py @@ -58,7 +58,7 @@ def gen(): r = self._print_row(row) if r is not None: yield r - except: + except Exception as e: get_structured_logger('server_error').error("Exception while executing printer", exception=e) self.result = -1 yield self._error(e) diff --git a/src/server/_validate.py b/src/server/_validate.py index 59e5aa7d0..175bcba72 100644 --- a/src/server/_validate.py +++ b/src/server/_validate.py @@ -98,6 +98,17 @@ def extract_integer(key: Union[str, Sequence[str]]) -> Optional[int]: raise ValidationFailedException(f"{key}: not a number: {s}") +def extract_float(key: Union[str, Sequence[str]]) -> Optional[float]: + s = _extract_value(key) + if not s: + # nothing to do + return None + try: + return float(s) + except ValueError as e: + raise ValidationFailedException(f"{key}: not a number: {s}") + + def extract_integers(key: Union[str, Sequence[str]]) -> Optional[List[IntRange]]: parts = extract_strings(key) if not parts: @@ -187,3 +198,13 @@ def push_range(first: str, last: str): values.append(parse_date(part)) # success, return the list return values + +def extract_bool(key: Union[str, Sequence[str]]) -> Optional[bool]: + s = _extract_value(key) + if not s: + return None + if s.lower() == "true": + return True + if s.lower() == "false": + return False + raise ValidationFailedException(f"{key}: not a boolean: {s}") diff --git a/src/server/endpoints/covidcast.py b/src/server/endpoints/covidcast.py index 2bd262e7e..0bbda43d9 100644 --- a/src/server/endpoints/covidcast.py +++ b/src/server/endpoints/covidcast.py @@ -1,14 +1,18 @@ from typing import List, Optional, Tuple, Dict, Any from itertools import groupby from datetime import date, timedelta +from bisect import bisect_right from epiweeks import Week from flask import Blueprint, request from flask.json import loads, jsonify -from bisect import bisect_right +from more_itertools import peekable +from numpy import nan from sqlalchemy import text from pandas import read_csv, to_datetime +from numbers import Number -from .._common import is_compatibility_mode, db +from .._common import is_compatibility_mode, app, db +from .._config import MAX_SMOOTHER_WINDOW from .._exceptions import ValidationFailedException, DatabaseErrorException from .._params import ( GeoPair, @@ -26,9 +30,11 @@ from .._query import QueryBuilder, execute_query, run_query, parse_row, filter_fields from .._printer import create_printer, CSVPrinter from .._validate import ( + extract_bool, extract_date, extract_dates, extract_integer, + extract_float, extract_strings, require_all, require_any, @@ -36,11 +42,12 @@ from .._pandas import as_pandas, print_pandas from .covidcast_utils import compute_trend, compute_trends, compute_correlations, compute_trend_value, CovidcastMetaEntry from ..utils import shift_day_value, day_to_time_value, time_value_to_iso, time_value_to_day, shift_week_value, time_value_to_week, guess_time_value_is_day, week_to_time_value, TimeValues -from .covidcast_utils.model import TimeType, count_signal_time_types, data_sources, create_source_signal_alias_mapper +from .covidcast_utils.model import TimeType, count_signal_time_types, data_sources, data_sources_by_id, create_source_signal_alias_mapper, get_pad_length, pad_time_pair, pad_time_window, get_basename_signal_and_jit_generator # first argument is the endpoint name bp = Blueprint("covidcast", __name__) alias = None +JIT_COMPUTE_ON = True latest_table = "epimetric_latest_v" history_table = "epimetric_full_v" @@ -58,7 +65,18 @@ def parse_source_signal_pairs() -> List[SourceSignalPair]: if ":" not in request.values.get("signal", ""): raise ValidationFailedException("missing parameter: signal or (data_source and signal[s])") - return parse_source_signal_arg() + # Convert source_signal_pairs with signal == True out to an explicit list of signals. + expanded_bool_source_signal_pairs = [] + for source_signal_pair in parse_source_signal_arg(): + if source_signal_pair.signal is True: + if data_source := data_sources_by_id.get(source_signal_pair.source): + expanded_bool_source_signal_pairs.append(SourceSignalPair(data_source.source, [s.signal for s in data_source.signals])) + else: + expanded_bool_source_signal_pairs.append(source_signal_pair) + else: + expanded_bool_source_signal_pairs.append(source_signal_pair) + + return expanded_bool_source_signal_pairs def parse_geo_pairs() -> List[GeoPair]: @@ -77,7 +95,7 @@ def parse_geo_pairs() -> List[GeoPair]: return parse_geo_arg() -def parse_time_pairs() -> TimePair: +def parse_time_pair() -> TimePair: time_type = request.values.get("time_type") if time_type: # old version @@ -88,7 +106,8 @@ def parse_time_pairs() -> TimePair: if ":" not in request.values.get("time", ""): raise ValidationFailedException("missing parameter: time or (time_type and time_values)") - return parse_time_arg() + time_pair = parse_time_arg() + return time_pair def _handle_lag_issues_as_of(q: QueryBuilder, issues: Optional[TimeValues] = None, lag: Optional[int] = None, as_of: Optional[int] = None): @@ -113,50 +132,121 @@ def _handle_lag_issues_as_of(q: QueryBuilder, issues: Optional[TimeValues] = Non pass +def parse_transform_args(): + # The length of the window to smooth over. + smoother_window_length = extract_integer("smoother_window_length") + if smoother_window_length is None: + smoother_window_length = 7 + elif not isinstance(smoother_window_length, int): + raise ValidationFailedException("smoother_window_length must be an integer") + elif smoother_window_length > MAX_SMOOTHER_WINDOW: + raise ValidationFailedException(f"smoother_window_length must be <= {MAX_SMOOTHER_WINDOW}") + + # The value to fill for missing date values. + pad_fill_value = extract_float("pad_fill_value") + if pad_fill_value is None: + pad_fill_value = nan + elif not isinstance(pad_fill_value, Number): + raise ValidationFailedException("pad_fill_value must be a number") + + # The value to fill for None or nan values. + nan_fill_value = extract_float("nans_fill_value") + if nan_fill_value is None: + nan_fill_value = nan + elif not isinstance(nan_fill_value, Number): + raise ValidationFailedException("nans_fill_value must be a number") + + smoother_args = { + "smoother_window_length": smoother_window_length, + "pad_fill_value": pad_fill_value, + "nans_fill_value": nan_fill_value, + } + return smoother_args + + +def parse_jit_bypass(): + jit_bypass = extract_bool("jit_bypass") + if jit_bypass is None: + return False + else: + return jit_bypass + + @bp.route("/", methods=("GET", "POST")) def handle(): source_signal_pairs = parse_source_signal_pairs() source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) - time_pair = parse_time_pairs() + time_pair = parse_time_pair() geo_pairs = parse_geo_pairs() + jit_bypass = parse_jit_bypass() as_of = extract_date("as_of") issues = extract_dates("issues") lag = extract_integer("lag") + is_time_type_week = time_pair.time_type == "week" + + is_compatibility = is_compatibility_mode() + def alias_row(row): + if is_compatibility: + # old api returned fewer fields + remove_fields = ["geo_type", "source", "time_type"] + for field in remove_fields: + if field in row: + del row[field] + if is_compatibility or not alias_mapper or "source" not in row: + return row + row["source"] = alias_mapper(row["source"], row["signal"]) + return row # build query q = QueryBuilder(latest_table, "t") - fields_string = ["geo_value", "signal"] + fields_string = ["geo_type", "geo_value", "source", "signal", "time_type"] fields_int = ["time_value", "direction", "issue", "lag", "missing_value", "missing_stderr", "missing_sample_size"] fields_float = ["value", "stderr", "sample_size"] - is_compatibility = is_compatibility_mode() - if is_compatibility: - q.set_order("signal", "time_value", "geo_value", "issue") + + use_jit_compute = not any((issues, lag, is_time_type_week)) and JIT_COMPUTE_ON and not jit_bypass + if use_jit_compute: + transform_args = parse_transform_args() + pad_length = get_pad_length(source_signal_pairs, transform_args.get("smoother_window_length")) + time_pair = pad_time_pair(time_pair, pad_length) + app.logger.info(f"JIT compute enabled for route '/': {source_signal_pairs}") + source_signal_pairs, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs, transform_args=transform_args) + app.logger.info(f"JIT base signals: {source_signal_pairs}") + + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + transformed_rows = row_transform_generator(parsed_rows, transform_args=transform_args) + for row in transformed_rows: + yield alias_row(row) else: - # transfer also the new detail columns - fields_string.extend(["source", "geo_type", "time_type"]) - q.set_order("source", "signal", "time_type", "time_value", "geo_type", "geo_value", "issue") + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + for row in parsed_rows: + yield alias_row(row) + + q.set_order("source", "signal", "geo_type", "geo_value", "time_type", "time_value", "issue") q.set_fields(fields_string, fields_int, fields_float) # basic query info # data type of each field # build the source, signal, time, and location (type and id) filters - q.where_source_signal_pairs("source", "signal", source_signal_pairs) q.where_geo_pairs("geo_type", "geo_value", geo_pairs) q.where_time_pair("time_type", "time_value", time_pair) _handle_lag_issues_as_of(q, issues, lag, as_of) - def transform_row(row, proxy): - if is_compatibility or not alias_mapper or "source" not in row: - return row - row["source"] = alias_mapper(row["source"], proxy["signal"]) - return row + p = create_printer() - # send query - return execute_query(str(q), q.params, fields_string, fields_int, fields_float, transform=transform_row) + # execute first query + try: + r = run_query(p, (str(q), q.params)) + except Exception as e: + raise DatabaseErrorException(str(e)) + + # now use a generator for sending the rows and execute all the other queries + return p(filter_fields(gen_transform(r))) def _verify_argument_time_type_matches(is_day_argument: bool, count_daily_signal: int, count_weekly_signal: int) -> None: @@ -173,6 +263,8 @@ def handle_trend(): daily_signals, weekly_signals = count_signal_time_types(source_signal_pairs) source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) geo_pairs = parse_geo_pairs() + transform_args = parse_transform_args() + jit_bypass = parse_jit_bypass() time_window = parse_day_or_week_range_arg("window") is_day = time_window.is_day @@ -180,7 +272,9 @@ def handle_trend(): time_value, is_also_day = time_pair.time_values[0], time_pair.is_day if is_day != is_also_day: raise ValidationFailedException("mixing weeks with day arguments") + _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) + basis_time_value = extract_date("basis") if basis_time_value is None: base_shift = extract_integer("basis_shift") @@ -188,14 +282,42 @@ def handle_trend(): base_shift = 7 basis_time_value = shift_day_value(time_value, -1 * base_shift) if is_day else shift_week_value(time_value, -1 * base_shift) + def gen_trend(rows): + for key, group in groupby(rows, lambda row: (row["source"], row["signal"], row["geo_type"], row["geo_value"])): + source, signal, geo_type, geo_value = key + if alias_mapper: + source = alias_mapper(source, signal) + trend = compute_trend(geo_type, geo_value, source, signal, time_value, basis_time_value, ((row["time_value"], row["value"]) for row in group)) + yield trend.asdict() + # build query q = QueryBuilder(latest_table, "t") fields_string = ["geo_type", "geo_value", "source", "signal"] fields_int = ["time_value"] fields_float = ["value"] + + use_jit_compute = all((is_day, is_also_day)) and JIT_COMPUTE_ON and not jit_bypass + if use_jit_compute: + pad_length = get_pad_length(source_signal_pairs, transform_args.get("smoother_window_length")) + app.logger.info(f"JIT compute enabled for route '/trend': {source_signal_pairs}") + source_signal_pairs, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs) + app.logger.info(f"JIT base signals: {source_signal_pairs}") + time_window = pad_time_window(time_window, pad_length) + + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + transformed_rows = row_transform_generator(parsed_rows, transform_args=transform_args) + for row in transformed_rows: + yield row + else: + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + for row in parsed_rows: + yield row + q.set_fields(fields_string, fields_int, fields_float) - q.set_order("geo_type", "geo_value", "source", "signal", "time_value") + q.set_order("source", "signal", "geo_type", "geo_value", "time_value") q.where_source_signal_pairs("source", "signal", source_signal_pairs) q.where_geo_pairs("geo_type", "geo_value", geo_pairs) @@ -206,13 +328,6 @@ def handle_trend(): p = create_printer() - def gen(rows): - for key, group in groupby((parse_row(row, fields_string, fields_int, fields_float) for row in rows), lambda row: (row["geo_type"], row["geo_value"], row["source"], row["signal"])): - geo_type, geo_value, source, signal = key - if alias_mapper: - source = alias_mapper(source, signal) - trend = compute_trend(geo_type, geo_value, source, signal, time_value, basis_time_value, ((row["time_value"], row["value"]) for row in group)) - yield trend.asdict() # execute first query try: @@ -221,7 +336,7 @@ def gen(rows): raise DatabaseErrorException(str(e)) # now use a generator for sending the rows and execute all the other queries - return p(filter_fields(gen(r))) + return p(filter_fields(gen_trend(gen_transform(r)))) @bp.route("/trendseries", methods=("GET", "POST")) @@ -231,22 +346,58 @@ def handle_trendseries(): daily_signals, weekly_signals = count_signal_time_types(source_signal_pairs) source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) geo_pairs = parse_geo_pairs() + transform_args = parse_transform_args() + jit_bypass = parse_jit_bypass() time_window = parse_day_or_week_range_arg("window") is_day = time_window.is_day _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) + basis_shift = extract_integer(("basis", "basis_shift")) if basis_shift is None: basis_shift = 7 + shifter = lambda x: shift_day_value(x, -basis_shift) + if not is_day: + shifter = lambda x: shift_week_value(x, -basis_shift) + + def gen_trend(rows): + for key, group in groupby(rows, lambda row: (row["source"], row["signal"], row["geo_type"], row["geo_value"])): + source, signal, geo_type, geo_value = key + if alias_mapper: + source = alias_mapper(source, signal) + trends = compute_trends(geo_type, geo_value, source, signal, shifter, ((row["time_value"], row["value"]) for row in group)) + for t in trends: + yield t.asdict() + # build query q = QueryBuilder(latest_table, "t") fields_string = ["geo_type", "geo_value", "source", "signal"] fields_int = ["time_value"] fields_float = ["value"] + + use_jit_compute = is_day and JIT_COMPUTE_ON and not jit_bypass + if use_jit_compute: + pad_length = get_pad_length(source_signal_pairs, transform_args.get("smoother_window_length")) + app.logger.info(f"JIT compute enabled for route '/trendseries': {source_signal_pairs}") + source_signal_pairs, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs) + app.logger.info(f"JIT base signals: {source_signal_pairs}") + time_window = pad_time_window(time_window, pad_length) + + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + transformed_rows = row_transform_generator(parsed_rows, transform_args=transform_args) + for row in transformed_rows: + yield row + else: + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + for row in parsed_rows: + yield row + q.set_fields(fields_string, fields_int, fields_float) - q.set_order("geo_type", "geo_value", "source", "signal", "time_value") + q.set_order("source", "signal", "geo_type", "geo_value", "time_value") q.where_source_signal_pairs("source", "signal", source_signal_pairs) q.where_geo_pairs("geo_type", "geo_value", geo_pairs) @@ -257,19 +408,6 @@ def handle_trendseries(): p = create_printer() - shifter = lambda x: shift_day_value(x, -basis_shift) - if not is_day: - shifter = lambda x: shift_week_value(x, -basis_shift) - - def gen(rows): - for key, group in groupby((parse_row(row, fields_string, fields_int, fields_float) for row in rows), lambda row: (row["geo_type"], row["geo_value"], row["source"], row["signal"])): - geo_type, geo_value, source, signal = key - if alias_mapper: - source = alias_mapper(source, signal) - trends = compute_trends(geo_type, geo_value, source, signal, shifter, ((row["time_value"], row["value"]) for row in group)) - for t in trends: - yield t.asdict() - # execute first query try: r = run_query(p, (str(q), q.params)) @@ -277,7 +415,7 @@ def gen(rows): raise DatabaseErrorException(str(e)) # now use a generator for sending the rows and execute all the other queries - return p(filter_fields(gen(r))) + return p(filter_fields(gen_trend(gen_transform(r)))) @bp.route("/correlation", methods=("GET", "POST")) @@ -363,10 +501,15 @@ def handle_export(): start_day, is_day = start_pair.time_values[0], start_pair.is_day end_pair = parse_day_or_week_arg("end_day", 202020 if weekly_signals > 0 else 20200901) end_day, is_end_day = end_pair.time_values[0], end_pair.is_day + time_window = TimePair("day" if is_day else "week", [(start_day, end_day)]) if is_day != is_end_day: raise ValidationFailedException("mixing weeks with day arguments") + _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) + transform_args = parse_transform_args() + jit_bypass = parse_jit_bypass() + geo_type = request.values.get("geo_type", "county") geo_values = request.values.get("geo_values", "*") @@ -380,10 +523,32 @@ def handle_export(): # build query q = QueryBuilder(latest_table, "t") - q.set_fields(["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "source"], [], []) - q.set_order("time_value", "geo_value") + fields_string = ["geo_value", "signal", "geo_type", "source"] + fields_int = ["time_value", "issue", "lag"] + fields_float = ["value", "stderr", "sample_size"] + + use_jit_compute = all([is_day, is_end_day]) and JIT_COMPUTE_ON and not jit_bypass + if use_jit_compute: + pad_length = get_pad_length(source_signal_pairs, transform_args.get("smoother_window_length")) + app.logger.info(f"JIT compute enabled for route '/csv': {source_signal_pairs}") + source_signal_pairs, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs) + app.logger.info(f"JIT base signals: {source_signal_pairs}") + time_window = pad_time_window(time_window, pad_length) + + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + transformed_rows = row_transform_generator(parsed_rows, transform_args=transform_args) + for row in transformed_rows: + yield row + else: + def gen_transform(rows): + for row in rows: + yield row + + q.set_fields(fields_string, fields_int, fields_float) + q.set_order("geo_value", "time_value") q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_time_pair("time_type", "time_value", TimePair("day" if is_day else "week", [(start_day, end_day)])) + q.where_time_pair("time_type", "time_value", time_window) q.where_geo_pairs("geo_type", "geo_value", [GeoPair(geo_type, True if geo_values == "*" else geo_values)]) _handle_lag_issues_as_of(q, None, None, as_of) @@ -394,7 +559,7 @@ def handle_export(): filename = "covidcast-{source}-{signal}-{start_day}-to-{end_day}{as_of}".format(source=source, signal=signal, start_day=format_date(start_day), end_day=format_date(end_day), as_of=as_of_str) p = CSVPrinter(filename) - def parse_row(i, row): + def parse_csv_row(i, row): # '',geo_value,signal,{time_value,issue},lag,value,stderr,sample_size,geo_type,data_source return { "": i, @@ -410,10 +575,9 @@ def parse_row(i, row): "data_source": alias_mapper(row["source"], row["signal"]) if alias_mapper else row["source"], } - def gen(first_row, rows): - yield parse_row(0, first_row) + def gen_parse(rows): for i, row in enumerate(rows): - yield parse_row(i + 1, row) + yield parse_csv_row(i, row) # execute query try: @@ -422,14 +586,15 @@ def gen(first_row, rows): raise DatabaseErrorException(str(e)) # special case for no data to be compatible with the CSV server - first_row = next(r, None) + transformed_query = peekable(gen_transform(r)) + first_row = transformed_query.peek(None) if not first_row: return "No matching data found for signal {source}:{signal} " "at {geo} level from {start_day} to {end_day}, as of {as_of}.".format( source=source, signal=signal, geo=geo_type, start_day=format_date(start_day), end_day=format_date(end_day), as_of=(date.today().isoformat() if as_of is None else format_date(as_of)) ) # now use a generator for sending the rows and execute all the other queries - return p(gen(first_row, r)) + return p(gen_parse(transformed_query)) @bp.route("/backfill", methods=("GET", "POST")) diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index 13a9b91d0..3d4cb0cb5 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -1,12 +1,21 @@ from dataclasses import asdict, dataclass, field -from typing import Callable, Optional, Dict, List, Set, Tuple +from datetime import timedelta from enum import Enum +from functools import partial +from itertools import groupby, repeat, tee +from numbers import Number +from typing import Callable, Generator, Iterator, Optional, Dict, List, Set, Tuple, Union + from pathlib import Path import re +from more_itertools import interleave_longest, peekable import pandas as pd import numpy as np -from ..._params import SourceSignalPair +from delphi_utils.nancodes import Nans +from ..._params import SourceSignalPair, TimePair +from .smooth_diff import generate_smoothed_rows, generate_diffed_rows +from ...utils import shift_day_value, day_to_time_value, time_value_to_day PANDAS_DTYPES = { @@ -30,6 +39,13 @@ "value_updated_timestamp": "Int64", } +IDENTITY: Callable = lambda rows, **kwargs: rows +DIFF: Callable = lambda rows, **kwargs: generate_diffed_rows(rows, **kwargs) +SMOOTH: Callable = lambda rows, **kwargs: generate_smoothed_rows(rows, **kwargs) +DIFF_SMOOTH: Callable = lambda rows, **kwargs: generate_smoothed_rows(generate_diffed_rows(rows, **kwargs), **kwargs) + +SignalTransforms = Dict[SourceSignalPair, SourceSignalPair] +TransformType = Callable[[Iterator[Dict]], Iterator[Dict]] class HighValuesAre(str, Enum): bad = "bad" @@ -43,6 +59,7 @@ class SignalFormat(str, Enum): fraction = "fraction" raw_count = "raw_count" raw = "raw" + count = "count" class SignalCategory(str, Enum): @@ -258,8 +275,8 @@ def _load_data_signals(sources: List[DataSource]): data_signals_by_key[(source.db_source, d.signal)] = d -def get_related_signals(signal: DataSignal) -> List[DataSignal]: - return [s for s in data_signals if s != signal and s.signal_basename == signal.signal_basename] +def get_related_signals(data_signal: DataSignal) -> List[DataSignal]: + return [s for s in data_signals if s != data_signal and s.signal_basename == data_signal.signal_basename] def count_signal_time_types(source_signals: List[SourceSignalPair]) -> Tuple[int, int]: @@ -321,3 +338,256 @@ def map_row(source: str, signal: str) -> str: return signal_source.source return transformed_pairs, map_row + + +def _reindex_iterable(iterator: Iterator[Dict], fill_value: Optional[Number] = None) -> Iterator[Dict]: + """Produces an iterator that fills in gaps in the time values of another iterator. + + Used to produce an iterator with a contiguous time index for time series operations. + The iterator is assumed to be sorted by time_value in ascending order. + The min and max time_values are determined from the first and last rows of the iterator. + The fill_value is used to fill in gaps in the time index. + """ + _iterator = peekable(iterator) + + # If the iterator is empty, we halt immediately. + try: + first_item = _iterator.peek() + except StopIteration: + return + + _default_item = first_item.copy() + _default_item.update({ + "stderr": None, + "sample_size": None, + "issue": None, + "lag": None, + "missing_stderr": Nans.NOT_APPLICABLE, + "missing_sample_size": Nans.NOT_APPLICABLE, + "id": None, + "direction": None, + "direction_updated_timestamp": None, + "value_updated_timestamp": None + }) + + expected_time_value = first_item["time_value"] + # Non-trivial operations otherwise. + while True: + try: + # This will stay the same until the peeked element is consumed. + new_item = _iterator.peek() + except StopIteration: + return + + if expected_time_value == new_item.get("time_value"): + # Get the value we just peeked. + yield next(_iterator) + else: + # Return a default row instead. + # Copy to avoid Python by-reference memory issues. + default_item = _default_item.copy() + default_item.update( + { + "time_value": expected_time_value, + "value": fill_value, + "missing_value": Nans.NOT_MISSING if pd.notna(fill_value) else Nans.NOT_APPLICABLE, + } + ) + yield default_item + expected_time_value = day_to_time_value(time_value_to_day(expected_time_value) + timedelta(days=1)) + + +def _get_base_signal_transform(signal: Union[DataSignal, Tuple[str, str]]) -> Callable: + """Given a DataSignal, return the transformation that needs to be applied to its base signal to derive the signal.""" + if isinstance(signal, DataSignal): + base_signal = data_signals_by_key.get((signal.source, signal.signal_basename)) + if signal.format not in [SignalFormat.raw, SignalFormat.raw_count, SignalFormat.count] or not signal.compute_from_base or not base_signal: + return IDENTITY + if signal.is_cumulative and signal.is_smoothed: + return SMOOTH + if not signal.is_cumulative and not signal.is_smoothed: + return DIFF if base_signal.is_cumulative else IDENTITY + if not signal.is_cumulative and signal.is_smoothed: + return DIFF_SMOOTH if base_signal.is_cumulative else SMOOTH + return IDENTITY + + if isinstance(signal, tuple): + if signal := data_signals_by_key.get(signal): + return _get_base_signal_transform(signal) + return IDENTITY + + raise TypeError("signal must be either Tuple[str, str] or DataSignal.") + + +def get_transform_types(source_signal_pairs: List[SourceSignalPair]) -> Set[Callable]: + """Return a collection of the unique transforms required for transforming a given source-signal pair list. + + Example: + SourceSignalPair("src", ["sig", "sig_smoothed", "sig_diff"]) would return {IDENTITY, SMOOTH, DIFF}. + + Used to pad the user DB query with extra days. + """ + transform_types = set() + for source_signal_pair in source_signal_pairs: + source_name = source_signal_pair.source + signal_names = source_signal_pair.signal + + if isinstance(signal_names, bool): + continue + + transform_types |= {_get_base_signal_transform((source_name, signal_name)) for signal_name in signal_names} + + return transform_types + + +def get_pad_length(source_signal_pairs: List[SourceSignalPair], smoother_window_length: int): + """Returns the size of the extra date padding needed, depending on the transformations the source-signal pair list requires. + + If smoothing is required, we fetch an extra smoother_window_length - 1 days (6 by default). If both diffing and smoothing is required on the same signal, + then we fetch extra smoother_window_length days (7 by default). + + Used to pad the user DB query with extra days. + """ + transform_types = get_transform_types(source_signal_pairs) + pad_length = [0] + if DIFF_SMOOTH in transform_types: + pad_length.append(smoother_window_length) + if SMOOTH in transform_types: + pad_length.append(smoother_window_length - 1) + if DIFF in transform_types: + pad_length.append(1) + return max(pad_length) + + +def pad_time_pair(time_pair: TimePair, pad_length: int) -> TimePair: + """Pads a list of TimePairs with another TimePair that extends the smallest time value by the pad_length, if needed. + + Assumes day time_type, since this function is only called for JIT computations which share the same assumption. + + Example: + [TimePair("day", [20210407])] with pad_length 6 would return [TimePair("day", [20210407]), TimePair("day", [(20210401, 20210407)])]. + """ + if pad_length < 0: + raise ValueError("pad_length should be a positive integer.") + + if pad_length == 0: + return time_pair + + if time_pair.time_type != "day": + raise ValueError("pad_time_pair assumes day time_type.") + + min_time = float("inf") + if not isinstance(time_pair.time_values, bool): + for time_value in time_pair.time_values: + min_time = min(min_time, time_value if isinstance(time_value, int) else time_value[0]) + + padded_time = (shift_day_value(min_time, -1 * pad_length), min_time) + time_pair = TimePair(time_pair.time_type, time_pair.time_values + [padded_time]) + + return time_pair + + +def pad_time_window(time_window: TimePair, pad_length: int) -> TimePair: + """Extend a TimePair with a single range time value on the left by pad_length. + + Example: + (20210407, 20210413) with pad_length 6 would return (20210401, 20210413). + + Used to pad the user DB query with extra days. + """ + if pad_length < 0: + raise ValueError("pad_length should non-negative.") + if pad_length == 0: + return time_window + if time_window.time_type != "day": + raise ValueError("pad_time_window assumes day time_type.") + if isinstance(time_window.time_values, bool): + return time_window + if len(time_window.time_values) != 1: + raise ValueError("pad_time_window assumes a single time value.") + min_time, max_time = time_window.time_values[0] + return TimePair("day", [(shift_day_value(min_time, -1 * pad_length), max_time)]) + + +def _generate_transformed_rows( + parsed_rows: Iterator[Dict], + transform_dict: Optional[SignalTransforms] = None, + transform_args: Optional[Dict] = None, + group_keyfunc: Optional[Callable] = None, +) -> Iterator[Dict]: + """Applies time-series transformations to streamed rows from a database. + + Parameters: + parsed_rows: Iterator[Dict] + An iterator streaming rows from a database query. Assumed to be sorted by source, signal, geo_type, geo_value, time_type, and time_value. + transform_dict: Optional[SignalTransforms], default None + A dictionary mapping base sources to a list of their derived signals that the user wishes to query. + For example, transform_dict may be {("jhu-csse", "confirmed_cumulative_num): [("jhu-csse", "confirmed_incidence_num"), ("jhu-csse", "confirmed_7dav_incidence_num")]}. + transform_args: Optional[Dict], default None + A dictionary of keyword arguments for the transformer functions. + group_keyfunc: Optional[Callable], default None + The groupby function to use to order the streamed rows. Note that Python groupby does not do any sorting, so + parsed_rows are assumed to be sorted in accord with this groupby. + + Yields: + transformed rows: Dict + The transformed rows returned in an interleaved fashion. Non-transformed rows have the IDENTITY operation applied. + """ + if not transform_args: + transform_args = dict() + if not transform_dict: + transform_dict = dict() + if not group_keyfunc: + group_keyfunc = lambda row: (row["source"], row["signal"], row["geo_type"], row["geo_value"]) + + for key, source_signal_geo_rows in groupby(parsed_rows, group_keyfunc): + base_source_name, base_signal_name, _, _ = key + # Extract the list of derived signals; if a signal is not in the dictionary, then use the identity map. + derived_signal_transform_map: SourceSignalPair = transform_dict.get(SourceSignalPair(base_source_name, [base_signal_name]), SourceSignalPair(base_source_name, [base_signal_name])) + # Create a list of source-signal pairs along with the transformation required for the signal. + signal_names_and_transforms: List[Tuple[Tuple[str, str], Callable]] = [(derived_signal, _get_base_signal_transform((base_source_name, derived_signal))) for derived_signal in derived_signal_transform_map.signal] + # Put the current time series on a contiguous time index. + source_signal_geo_rows = _reindex_iterable(source_signal_geo_rows, fill_value=transform_args.get("pad_fill_value")) + # Create copies of the iterable, with smart memory usage. + source_signal_geo_rows_copies: Iterator[Iterator[Dict]] = tee(source_signal_geo_rows, len(signal_names_and_transforms)) + # Create a list of transformed group iterables, remembering their derived name as needed. + transformed_signals_iterator: Iterator[Tuple[str, Iterator[Dict]]] = (zip(repeat(derived_signal), transform(rows, **transform_args)) for (derived_signal, transform), rows in zip(signal_names_and_transforms, source_signal_geo_rows_copies)) + # Traverse through the transformed iterables in an interleaved fashion, which makes sure that only a small window + # of the original iterable (group) is stored in memory. + for derived_signal_name, row in interleave_longest(*transformed_signals_iterator): + row["signal"] = derived_signal_name + yield row + + +def get_basename_signal_and_jit_generator(source_signal_pairs: List[SourceSignalPair], transform_args: Optional[Dict[str, Union[str, int]]] = None) -> Tuple[List[SourceSignalPair], Generator]: + """From a list of SourceSignalPairs, return the base signals required to derive them and a transformation function to take a stream + of the base signals and return the transformed signals. + + Example: + SourceSignalPair("src", signal=["sig_base", "sig_smoothed"]) would return SourceSignalPair("src", signal=["sig_base"]) and a transformation function + that will take the returned database query for "sig_base" and return both the base time series and the smoothed time series. transform_dict in this case + would be {("src", "sig_base"): [("src", "sig_base"), ("src", "sig_smooth")]}. + """ + base_signal_pairs: List[SourceSignalPair] = [] + transform_dict: SignalTransforms = dict() + + for pair in source_signal_pairs: + # Should only occur when the SourceSignalPair was unrecognized by _resolve_bool_source_signals. Useful for testing with fake signal names. + if isinstance(pair.signal, bool): + base_signal_pairs.append(pair) + continue + + signals = [] + for signal_name in pair.signal: + signal = data_signals_by_key.get((pair.source, signal_name)) + if not signal or not signal.compute_from_base: + transform_dict.setdefault(SourceSignalPair(source=pair.source, signal=[signal_name]), SourceSignalPair(source=pair.source, signal=[])).add_signal(signal_name) + signals.append(signal_name) + else: + transform_dict.setdefault(SourceSignalPair(source=pair.source, signal=[signal.signal_basename]), SourceSignalPair(source=pair.source, signal=[])).add_signal(signal_name) + signals.append(signal.signal_basename) + base_signal_pairs.append(SourceSignalPair(pair.source, signals)) + + row_transform_generator = partial(_generate_transformed_rows, transform_dict=transform_dict, transform_args=transform_args) + + return base_signal_pairs, row_transform_generator diff --git a/src/server/endpoints/covidcast_utils/smooth_diff.py b/src/server/endpoints/covidcast_utils/smooth_diff.py new file mode 100644 index 000000000..80622da26 --- /dev/null +++ b/src/server/endpoints/covidcast_utils/smooth_diff.py @@ -0,0 +1,136 @@ +from enum import Enum +from logging import getLogger +from numbers import Number +from typing import Dict, Iterable, List + +import numpy as np +from delphi_utils.nancodes import Nans +from more_itertools import windowed + +from ...utils.dates import time_value_to_day + + +class SmootherKernelValue(str, Enum): + average = "average" + + +def generate_smoothed_rows( + rows: Iterable[Dict], + smoother_window_length: int = 7, + nan_fill_value: Number = np.nan, + **kwargs +) -> Iterable[Dict]: + """Generate smoothed row entries. + + There are roughly two modes of boundary handling: + * no padding, the windows start at length 1 on the left boundary and grow to size + smoother_window_length (achieved with pad_fill_value = None) + * value padding, smoother_window_length - 1 many fill_values are appended at the start of the + given date (achieved with any other pad_fill_value) + + Note that this function crucially relies on the assumption that the iterable rows + have been sorted by time_value. If this assumption is violated, the results will likely be + incoherent. + + Parameters + ---------- + rows: Iterable[Dict] + An iterable over the rows a database query returns. The rows are assumed to be + dicts containing the "geo_type", "geo_value", and "time_value" keys. Assumes the + rows have been sorted by geo and time_value beforehand. + smoother_window_length: int, default 7 + The length of the averaging window for the smoother. + nan_fill_value: Number, default nan + The value to use when encountering nans (e.g. None and numpy.nan types); uses nan by default. + **kwargs: + Container for non-shared parameters with other computation functions. + """ + # Validate params. + if not isinstance(smoother_window_length, int) or smoother_window_length < 1: + smoother_window_length = 7 + if not isinstance(nan_fill_value, Number): + nan_fill_value = np.nan + + for window in windowed(rows, smoother_window_length): # Iterable[List[Dict]] + # This occurs only if len(rows) < smoother_window_length. + if None in window: + continue + + new_value = 0 + for row in window: + if row.get("value") is not None: + new_value += row["value"] + else: + new_value += nan_fill_value + new_value /= smoother_window_length + new_value = float(round(new_value, 7)) if not np.isnan(new_value) else None + + new_item = _fill_remaining_row_fields(window) + new_item.update({"value": new_value, "missing_value": Nans.NOT_MISSING if new_value is not None else Nans.NOT_APPLICABLE}) + + yield new_item + + +def generate_diffed_rows(rows: Iterable[Dict], nan_fill_value: Number = np.nan, **kwargs) -> Iterable[Dict]: + """Generate differences between row values. + + Note that this function crucially relies on the assumption that the iterable rows have been + sorted by time_value. If this assumption is violated, the results will likely be incoherent. + + rows: Iterable[Dict] + An iterable over the rows a database query returns. The rows are assumed to be dicts + containing the "geo_type", "geo_value", and "time_value" keys. Assumes the rows have been + sorted by geo and time_value beforehand. + nan_fill_value: Number, default nan + The value to use when encountering nans (e.g. None and numpy.nan types); uses nan by default. + **kwargs: + Container for non-shared parameters with other computation functions. + """ + if not isinstance(nan_fill_value, Number): + nan_fill_value = np.nan + + for window in windowed(rows, 2): + # This occurs only if len(rows) < 2. + if None in window: + continue + + first_value = window[0]["value"] if window[0].get("value") is not None else nan_fill_value + second_value = window[1]["value"] if window[1].get("value") is not None else nan_fill_value + new_value = float(round(second_value - first_value, 7)) + new_value = new_value if not np.isnan(new_value) else None + + new_item = _fill_remaining_row_fields(window) + new_item.update({"value": new_value, "missing_value": Nans.NOT_MISSING if new_value is not None else Nans.NOT_APPLICABLE}) + + yield new_item + + +def _fill_remaining_row_fields(window: List[dict]) -> dict: + """Set a few default fields for the covidcast row.""" + logger = getLogger("gunicorn.error") + + # Start by defaulting to the field values of the last window member. + new_item = window[-1].copy() + + try: + new_issue = max(e.get("issue") for e in window if e.get("issue") is not None) + except (TypeError, ValueError): + logger.warn(f"There was an error computing an issue field for {new_item.get('source')}:{new_item.get('signal')}, geo: {new_item.get('geo_value')}, date: {new_item.get('time_value')}.") + new_issue = None + + try: + new_lag = (time_value_to_day(new_issue) - time_value_to_day(new_item["time_value"])).days + except (TypeError, ValueError): + logger.warn(f"There was an error computing a lag field for {new_item.get('source')}:{new_item.get('signal')}, geo: {new_item.get('geo_value')}, date: {new_item.get('time_value')}.") + new_lag = None + + new_item.update({ + "issue": new_issue, + "lag": new_lag, + "stderr": None, + "sample_size": None, + "missing_stderr": Nans.NOT_APPLICABLE, + "missing_sample_size": Nans.NOT_APPLICABLE + }) + + return new_item diff --git a/src/server/endpoints/covidcast_utils/test_utils.py b/src/server/endpoints/covidcast_utils/test_utils.py new file mode 100644 index 000000000..668a4a8fc --- /dev/null +++ b/src/server/endpoints/covidcast_utils/test_utils.py @@ -0,0 +1,176 @@ +import numpy as np +import pandas as pd + +from delphi_utils.nancodes import Nans +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRows, set_df_dtypes +from delphi.epidata.server.endpoints.covidcast_utils.model import DataSource, DataSignal +from delphi.epidata.server.utils.dates import iterate_over_range + + +# fmt: off +DATA_SIGNALS_BY_KEY = { + ("src", "sig_diff"): DataSignal( + source="src", + signal="sig_diff", + signal_basename="sig_base", + name="src", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=False, + compute_from_base=True, + ), + ("src", "sig_smooth"): DataSignal( + source="src", + signal="sig_smooth", + signal_basename="sig_base", + name="src", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=True, + is_smoothed=True, + compute_from_base=True, + ), + ("src", "sig_diff_smooth"): DataSignal( + source="src", + signal="sig_diff_smooth", + signal_basename="sig_base", + name="src", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=False, + is_smoothed=True, + compute_from_base=True, + ), + ("src", "sig_base"): DataSignal( + source="src", + signal="sig_base", + signal_basename="sig_base", + name="src", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=True, + ), + ("src2", "sig_base"): DataSignal( + source="src2", + signal="sig_base", + signal_basename="sig_base", + name="sig_base", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=True, + ), + ("src2", "sig_diff_smooth"): DataSignal( + source="src2", + signal="sig_diff_smooth", + signal_basename="sig_base", + name="sig_smooth", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=False, + is_smoothed=True, + compute_from_base=True, + ), +} + +DATA_SOURCES_BY_ID = { + "src": DataSource( + source="src", + db_source="src", + name="src", + description="", + reference_signal="sig_base", + signals=[DATA_SIGNALS_BY_KEY[key] for key in DATA_SIGNALS_BY_KEY if key[0] == "src"], + ), + "src2": DataSource( + source="src2", + db_source="src2", + name="src2", + description="", + reference_signal="sig_base", + signals=[DATA_SIGNALS_BY_KEY[key] for key in DATA_SIGNALS_BY_KEY if key[0] == "src2"], + ), +} +# fmt: on + + +# A slow JIT method to sanity check values. +def reindex_df(df: pd.DataFrame) -> pd.DataFrame: + dfs = [] + for key, group_df in df.groupby(["source", "signal", "geo_value"]): + group_df = group_df.set_index("time_value").sort_index() + group_df = group_df.reindex(iterate_over_range(group_df.index.min(), group_df.index.max(), inclusive=True)) + group_df["source"] = group_df["source"].ffill() + group_df["signal"] = group_df["signal"].ffill() + group_df["geo_value"] = group_df["geo_value"].ffill() + group_df["geo_type"] = group_df["geo_type"].ffill() + group_df["time_type"] = group_df["time_type"].ffill() + group_df["missing_value"] = np.where(group_df["value"].isna(), Nans.NOT_APPLICABLE, Nans.NOT_MISSING) + group_df["missing_stderr"] = np.where(group_df["stderr"].isna(), Nans.NOT_APPLICABLE, Nans.NOT_MISSING) + group_df["missing_sample_size"] = np.where(group_df["sample_size"].isna(), Nans.NOT_APPLICABLE, Nans.NOT_MISSING) + dfs.append(group_df.reset_index()) + ndf = pd.concat(dfs) + ndf = set_df_dtypes(ndf, CovidcastRows._pandas_dtypes) + return ndf + +def diff_df(df: pd.DataFrame, signal_name: str, nan_fill_value: float = np.nan, omit_left_boundary: bool = False) -> pd.DataFrame: + df = reindex_df(df) + dfs = [] + for key, group_df in df.groupby(["source", "signal", "geo_value"]): + group_df = group_df.set_index("time_value").sort_index() + group_df["value"] = group_df["value"].fillna(nan_fill_value).diff() + group_df["stderr"] = np.nan + group_df["sample_size"] = np.nan + group_df["missing_value"] = np.where(group_df["value"].isna(), Nans.NOT_APPLICABLE, Nans.NOT_MISSING) + group_df["missing_stderr"] = Nans.NOT_APPLICABLE + group_df["missing_sample_size"] = Nans.NOT_APPLICABLE + group_df["issue"] = group_df["issue"].rolling(2).max() + group_df["lag"] = group_df["lag"].rolling(2).max() + group_df["signal"] = signal_name + if omit_left_boundary: + group_df = group_df.iloc[1:] + dfs.append(group_df.reset_index()) + ndf = pd.concat(dfs) + ndf = set_df_dtypes(ndf, CovidcastRows._pandas_dtypes) + return ndf + +def smooth_df(df: pd.DataFrame, signal_name: str, nan_fill_value: float = np.nan, window_length: int = 7, omit_left_boundary: bool = False) -> pd.DataFrame: + df = reindex_df(df) + dfs = [] + for key, group_df in df.groupby(["source", "signal", "geo_value"]): + group_df = group_df.set_index("time_value").sort_index() + group_df["value"] = group_df["value"].fillna(nan_fill_value).rolling(window_length).mean() + group_df["stderr"] = np.nan + group_df["sample_size"] = np.nan + group_df["missing_value"] = np.where(group_df["value"].isna(), Nans.NOT_APPLICABLE, Nans.NOT_MISSING) + group_df["missing_stderr"] = Nans.NOT_APPLICABLE + group_df["missing_sample_size"] = Nans.NOT_APPLICABLE + group_df["issue"] = group_df["issue"].rolling(7).max() + group_df["lag"] = group_df["lag"].rolling(7).max() + group_df["signal"] = signal_name + if omit_left_boundary: + group_df = group_df.iloc[window_length - 1:] + dfs.append(group_df.reset_index()) + ndf = pd.concat(dfs) + ndf = set_df_dtypes(ndf, CovidcastRows._pandas_dtypes) + return ndf + +def diff_smooth_df(df: pd.DataFrame, signal_name: str, nan_fill_value: float = np.nan, window_length: int = 7, omit_left_boundary: bool = False) -> pd.DataFrame: + return smooth_df(diff_df(df, signal_name, nan_fill_value=nan_fill_value, omit_left_boundary=omit_left_boundary), signal_name, nan_fill_value=nan_fill_value, window_length=window_length, omit_left_boundary=omit_left_boundary) diff --git a/src/server/endpoints/covidcast_utils/trend.py b/src/server/endpoints/covidcast_utils/trend.py index 43c4ac21b..9a2825208 100644 --- a/src/server/endpoints/covidcast_utils/trend.py +++ b/src/server/endpoints/covidcast_utils/trend.py @@ -42,6 +42,8 @@ def compute_trend(geo_type: str, geo_value: str, signal_source: str, signal_sign # find all needed rows for time, value in rows: + if value is None: + continue if time == current_time: t.value = value if time == basis_time: @@ -73,6 +75,8 @@ def compute_trends(geo_type: str, geo_value: str, signal_source: str, signal_sig lookup: Dict[int, float] = OrderedDict() # find all needed rows for time, value in rows: + if value is None: + continue lookup[time] = value if min_value is None or min_value > value: min_date = time diff --git a/src/server/utils/__init__.py b/src/server/utils/__init__.py index efab6c030..868709b5a 100644 --- a/src/server/utils/__init__.py +++ b/src/server/utils/__init__.py @@ -1 +1,19 @@ -from .dates import shift_day_value, day_to_time_value, time_value_to_iso, time_value_to_day, days_in_range, weeks_in_range, shift_week_value, week_to_time_value, time_value_to_week, guess_time_value_is_day, guess_time_value_is_week, time_values_to_ranges, days_to_ranges, weeks_to_ranges, TimeValues +from .dates import ( + shift_day_value, + day_to_time_value, + time_value_to_iso, + time_value_to_day, + days_in_range, + weeks_in_range, + shift_week_value, + week_to_time_value, + time_value_to_week, + guess_time_value_is_day, + guess_time_value_is_week, + time_values_to_ranges, + days_to_ranges, + weeks_to_ranges, + iterate_over_range, + iterate_over_ints_and_ranges, + TimeValues, +) diff --git a/src/server/utils/dates.py b/src/server/utils/dates.py index b85465bb8..fa4f4eb1e 100644 --- a/src/server/utils/dates.py +++ b/src/server/utils/dates.py @@ -1,6 +1,7 @@ from datetime import date, timedelta from typing import ( Callable, + Iterator, Optional, Sequence, Tuple, @@ -48,6 +49,9 @@ def week_to_time_value(w: Week) -> int: def time_value_to_iso(value: int) -> str: return time_value_to_day(value).strftime("%Y-%m-%d") +def iso_to_time_value(iso: str) -> int: + return day_to_time_value(date.fromisoformat(iso)) + def shift_day_value(time_value: int, days: int) -> int: if days == 0: return time_value @@ -152,3 +156,50 @@ def _to_ranges(values: TimeValues, value_to_date: Callable, date_to_value: Calla except Exception as e: get_structured_logger('server_utils').error('bad input to date ranges', time_values=values, exception=e) return values + +def iterate_over_range(start: int, end: int, inclusive: bool = False) -> Iterator[int]: + """Iterate over ints corresponding to dates in a time range. + + By default left inclusive, right exclusive to mimic the behavior of the built-in range. + """ + if start > end: + return + + current_date, final_date = time_value_to_day(start), time_value_to_day(end) + while current_date < final_date: + yield day_to_time_value(current_date) + current_date = current_date + timedelta(days=1) + + if inclusive: + yield day_to_time_value(current_date) + +def iterate_over_ints_and_ranges(lst: Iterator[Union[int, Tuple[int, int]]], use_dates: bool = True) -> Iterator[int]: + """A generator that iterates over the unique values in a list of integers and ranges in ascending order. + + The tuples are assumed to be left- and right-inclusive. If use_dates is True, then the integers are interpreted as + YYYYMMDD dates. + + Examples: + >>> list(iterate_over_ints_and_ranges([(5, 8), 0], False)) + [0, 5, 6, 7, 8] + >>> list(iterate_over_ints_and_ranges([(5, 8), (4, 6), (3, 5)], False)) + [3, 4, 5, 6, 7, 8] + >>> list(iterate_over_ints_and_ranges([(7, 8), (5, 7), (3, 8), 8], False)) + [3, 4, 5, 6, 7, 8] + """ + lst = sorted((x, x) if isinstance(x, int) else x for x in lst) + if not lst: + return + + if use_dates: + increment = lambda x, y: day_to_time_value(time_value_to_day(x) + timedelta(days=y)) + range_handler = iterate_over_range + else: + increment = lambda x, y: x + y + range_handler = range + + biggest_seen = increment(lst[0][0], -1) + for a, b in lst: + for y in range_handler(max(a, increment(biggest_seen, 1)), increment(b, 1)): + yield y + biggest_seen = max(biggest_seen, b) diff --git a/tests/server/endpoints/covidcast_utils/test_model.py b/tests/server/endpoints/covidcast_utils/test_model.py new file mode 100644 index 000000000..efeffbc59 --- /dev/null +++ b/tests/server/endpoints/covidcast_utils/test_model.py @@ -0,0 +1,228 @@ +import unittest +from itertools import chain +from unittest.mock import patch + +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRows, assert_frame_equal_no_order +from delphi.epidata.server._params import SourceSignalPair, TimePair +from delphi.epidata.server.endpoints.covidcast_utils.model import ( + DIFF, + DIFF_SMOOTH, + IDENTITY, + SMOOTH, + _generate_transformed_rows, + _get_base_signal_transform, + _reindex_iterable, + get_basename_signal_and_jit_generator, + get_pad_length, + get_transform_types, + pad_time_pair, +) +from delphi.epidata.server.endpoints.covidcast_utils.test_utils import DATA_SOURCES_BY_ID, DATA_SIGNALS_BY_KEY, reindex_df, diff_df, smooth_df, diff_smooth_df + + +@patch("delphi.epidata.server.endpoints.covidcast_utils.model.data_sources_by_id", DATA_SOURCES_BY_ID) +@patch("delphi.epidata.server.endpoints.covidcast_utils.model.data_signals_by_key", DATA_SIGNALS_BY_KEY) +class TestModel(unittest.TestCase): + def test__reindex_iterable(self): + with self.subTest(f"Identity operations."): + assert list(_reindex_iterable([])) == [] + + data = CovidcastRows.from_args(time_value=pd.date_range("2021-05-03", "2021-05-08").to_list()) + df = CovidcastRows.from_records(_reindex_iterable(data.as_dicts())).db_row_df + assert_frame_equal(df, data.db_row_df) + + with self.subTest("Non-trivial operations"): + data = CovidcastRows.from_args(time_value=pd.date_range("2021-05-03", "2021-05-08").to_list() + pd.date_range("2021-05-11", "2021-05-14").to_list()) + df = CovidcastRows.from_records(_reindex_iterable(data.as_dicts())).db_row_df + expected_df = reindex_df(data.db_row_df) + assert_frame_equal_no_order(df, expected_df, index=["source", "signal", "geo_value", "time_value"]) + + def test__get_base_signal_transform(self): + assert _get_base_signal_transform(("src", "sig_smooth")) == SMOOTH + assert _get_base_signal_transform(("src", "sig_diff_smooth")) == DIFF_SMOOTH + assert _get_base_signal_transform(("src", "sig_diff")) == DIFF + assert _get_base_signal_transform(("src", "sig_diff")) == DIFF + assert _get_base_signal_transform(("src", "sig_base")) == IDENTITY + assert _get_base_signal_transform(("src", "sig_unknown")) == IDENTITY + + def test_get_transform_types(self): + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_diff"])] + transform_types = get_transform_types(source_signal_pairs) + expected_transform_types = {DIFF} + assert transform_types == expected_transform_types + + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_smooth"])] + transform_types = get_transform_types(source_signal_pairs) + expected_transform_types = {SMOOTH} + assert transform_types == expected_transform_types + + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_diff_smooth"])] + transform_types = get_transform_types(source_signal_pairs) + expected_transform_types = {DIFF_SMOOTH} + assert transform_types == expected_transform_types + + def test_get_pad_length(self): + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_diff"])] + pad_length = get_pad_length(source_signal_pairs, smoother_window_length=7) + assert pad_length == 1 + + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_smooth"])] + pad_length = get_pad_length(source_signal_pairs, smoother_window_length=5) + assert pad_length == 4 + + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_diff_smooth"])] + pad_length = get_pad_length(source_signal_pairs, smoother_window_length=10) + assert pad_length == 10 + + def test_pad_time_pair(self): + # fmt: off + time_pair = TimePair("day", [20210817, (20210810, 20210815)]) + expected_padded_time_pairs = TimePair("day", [20210817, (20210810, 20210815), (20210803, 20210810)]) + assert pad_time_pair(time_pair, pad_length=7) == expected_padded_time_pairs + + time_pairs = TimePair("day", True) + expected_padded_time_pairs = TimePair("day", True) + assert pad_time_pair(time_pairs, pad_length=7) == expected_padded_time_pairs + + time_pair = TimePair("day", [20210817, (20210810, 20210815)]) + expected_padded_time_pairs = TimePair("day", [20210817, (20210810, 20210815), (20210802, 20210810)]) + assert pad_time_pair(time_pair, pad_length=8) == expected_padded_time_pairs + + time_pairs = TimePair("day", [20210817, (20210810, 20210815)]) + assert pad_time_pair(time_pairs, pad_length=0) == time_pairs + # fmt: on + + def test__generate_transformed_rows(self): + # fmt: off + with self.subTest("diffed signal test"): + data = CovidcastRows.from_args( + signal=["sig_base"] * 5, + time_value=range(20210501, 20210506), + value=range(5) + ) + transform_dict = {SourceSignalPair("src", ["sig_base"]): SourceSignalPair("src", ["sig_diff"])} + df = CovidcastRows.from_records(_generate_transformed_rows(data.as_dicts(), transform_dict=transform_dict)).db_row_df + + expected_df = diff_df(data.db_row_df, "sig_diff", omit_left_boundary=True) + assert_frame_equal_no_order(df, expected_df, index=["signal", "geo_value", "time_value"]) + + with self.subTest("smoothed and diffed signals on one base test"): + data = CovidcastRows.from_args( + signal=["sig_base"] * 10, + time_value=pd.date_range("2021-05-01", "2021-05-10"), + value=range(10), + stderr=range(10), + sample_size=range(10) + ) + transform_dict = {SourceSignalPair("src", ["sig_base"]): SourceSignalPair("src", ["sig_diff", "sig_smooth"])} + df = CovidcastRows.from_records(_generate_transformed_rows(data.as_dicts(), transform_dict=transform_dict)).db_row_df + + expected_df = pd.concat([diff_df(data.db_row_df, "sig_diff", omit_left_boundary=True), smooth_df(data.db_row_df, "sig_smooth", omit_left_boundary=True)]) + assert_frame_equal_no_order(df, expected_df, index=["signal", "geo_value", "time_value"]) + + with self.subTest("smoothed and diffed signal on two non-continguous regions"): + data = CovidcastRows.from_args( + signal=["sig_base"] * 15, + time_value=chain(pd.date_range("2021-05-01", "2021-05-10"), pd.date_range("2021-05-16", "2021-05-20")), + value=range(15), + stderr=range(15), + sample_size=range(15), + ) + transform_dict = {SourceSignalPair("src", ["sig_base"]): SourceSignalPair("src", ["sig_diff", "sig_smooth"])} + df = CovidcastRows.from_records( + _generate_transformed_rows(data.as_dicts(), transform_dict=transform_dict) + ).db_row_df + + expected_df = pd.concat([diff_df(data.db_row_df, "sig_diff", omit_left_boundary=True), smooth_df(data.db_row_df, "sig_smooth", omit_left_boundary=True)]) + compare_cols = ["signal", "geo_value", "time_value", "time_type", "geo_type", "value", "stderr", "sample_size", "missing_value", "missing_stderr", "missing_sample_size", "direction"] + assert_frame_equal_no_order(df[compare_cols], expected_df[compare_cols], index=["signal", "geo_value", "time_value"]) + + with self.subTest("smooth_diffed signal on two non-continguous regions"): + data = CovidcastRows.from_args( + signal=["sig_base"] * 15, + time_value=chain(pd.date_range("2021-05-01", "2021-05-10"), pd.date_range("2021-05-16", "2021-05-20")), + value=range(15), + stderr=range(15), + sample_size=range(15), + ) + transform_dict = {SourceSignalPair("src", ["sig_base"]): SourceSignalPair("src", ["sig_diff_smooth"])} + df = CovidcastRows.from_records( + _generate_transformed_rows(data.as_dicts(), transform_dict=transform_dict) + ).db_row_df + + expected_df = diff_smooth_df(data.db_row_df, "sig_diff_smooth", omit_left_boundary=True) + compare_cols = ["signal", "geo_value", "time_value", "time_type", "geo_type", "value", "stderr", "sample_size", "missing_value", "missing_stderr", "missing_sample_size", "direction"] + assert_frame_equal_no_order(df[compare_cols], expected_df[compare_cols], index=["signal", "geo_value", "time_value"]) + # fmt: on + + def test_get_basename_signals(self): + with self.subTest("none to transform"): + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_base"])] + basename_pairs, _ = get_basename_signal_and_jit_generator(source_signal_pairs) + expected_basename_pairs = [SourceSignalPair(source="src", signal=["sig_base"])] + assert basename_pairs == expected_basename_pairs + + with self.subTest("unrecognized signal"): + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_unknown"])] + basename_pairs, _ = get_basename_signal_and_jit_generator(source_signal_pairs) + expected_basename_pairs = [SourceSignalPair(source="src", signal=["sig_unknown"])] + assert basename_pairs == expected_basename_pairs + + with self.subTest("plain"): + source_signal_pairs = [ + SourceSignalPair(source="src", signal=["sig_diff", "sig_smooth", "sig_diff_smooth", "sig_base"]), + SourceSignalPair(source="src2", signal=["sig"]), + ] + basename_pairs, _ = get_basename_signal_and_jit_generator(source_signal_pairs) + expected_basename_pairs = [ + SourceSignalPair(source="src", signal=["sig_base", "sig_base", "sig_base", "sig_base"]), + SourceSignalPair(source="src2", signal=["sig"]), + ] + assert basename_pairs == expected_basename_pairs + + with self.subTest("test base, diff, smooth"): + # fmt: off + data = CovidcastRows.from_args( + signal=["sig_base"] * 20 + ["sig_other"] * 5, + time_value=chain(pd.date_range("2021-05-01", "2021-05-10"), pd.date_range("2021-05-21", "2021-05-30"), pd.date_range("2021-05-01", "2021-05-05")), + value=chain(range(20), range(5)), + stderr=chain(range(20), range(5)), + sample_size=chain(range(20), range(5)), + ) + source_signal_pairs = [SourceSignalPair("src", ["sig_base", "sig_diff", "sig_other", "sig_smooth"])] + _, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs) + df = CovidcastRows.from_records(row_transform_generator(data.as_dicts())).db_row_df + + data_df = data.db_row_df + expected_df = pd.concat([reindex_df(data_df), diff_df(data_df[data_df["signal"] == "sig_base"], "sig_diff", omit_left_boundary=True), smooth_df(data_df[data_df["signal"] == "sig_base"], "sig_smooth", omit_left_boundary=True)]) + compare_cols = ["signal", "geo_value", "time_value", "time_type", "geo_type", "value", "stderr", "sample_size", "missing_value", "missing_stderr", "missing_sample_size", "direction"] + assert_frame_equal_no_order(df[compare_cols], expected_df[compare_cols], index=["signal", "geo_value", "time_value"]) + # fmt: on + + with self.subTest("test base, diff, smooth; multiple geos"): + # fmt: off + data = CovidcastRows.from_args( + signal=["sig_base"] * 40, + geo_value=["ak"] * 20 + ["ca"] * 20, + time_value=chain(pd.date_range("2021-05-01", "2021-05-20"), pd.date_range("2021-05-01", "2021-05-20")), + value=chain(range(20), range(0, 40, 2)), + stderr=chain(range(20), range(0, 40, 2)), + sample_size=chain(range(20), range(0, 40, 2)), + ) + source_signal_pairs = [SourceSignalPair("src", ["sig_base", "sig_diff", "sig_other", "sig_smooth"])] + _, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs) + df = CovidcastRows.from_records(row_transform_generator(data.as_dicts())).db_row_df + + expected_df = pd.concat([reindex_df(data.db_row_df), diff_df(data.db_row_df, "sig_diff", omit_left_boundary=True), smooth_df(data.db_row_df, "sig_smooth", omit_left_boundary=True)]) + compare_cols = ["signal", "geo_value", "time_value", "time_type", "geo_type", "value", "stderr", "sample_size", "missing_value", "missing_stderr", "missing_sample_size", "direction"] + assert_frame_equal_no_order(df[compare_cols], expected_df[compare_cols], index=["signal", "geo_value", "time_value"]) + # fmt: on + + with self.subTest("empty iterator"): + source_signal_pairs = [SourceSignalPair("src", ["sig_base", "sig_diff", "sig_smooth"])] + _, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs) + assert list(row_transform_generator({})) == [] diff --git a/tests/server/endpoints/covidcast_utils/test_smooth_diff.py b/tests/server/endpoints/covidcast_utils/test_smooth_diff.py new file mode 100644 index 000000000..16b6b9e72 --- /dev/null +++ b/tests/server/endpoints/covidcast_utils/test_smooth_diff.py @@ -0,0 +1,64 @@ +import unittest +from itertools import chain + +import pandas as pd + +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRows, assert_frame_equal_no_order +from delphi.epidata.server.endpoints.covidcast_utils.smooth_diff import generate_diffed_rows, generate_smoothed_rows +from delphi.epidata.server.endpoints.covidcast_utils.test_utils import diff_df, smooth_df + + +class TestStreaming(unittest.TestCase): + def test_generate_smoothed_rows(self): + data = pd.DataFrame({}) + with self.subTest("an empty dataframe should return an empty dataframe"): + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.to_dict(orient="records"))).api_row_df + expected_df = CovidcastRows(rows=[]).api_row_df + assert_frame_equal_no_order(smoothed_df, expected_df, index=["signal", "geo_value", "time_value"]) + + data = CovidcastRows.from_args(time_value=[20210501] * 6, value=[1.0] * 6) + with self.subTest("a dataframe with not enough entries to make a single smoothed value, should return an empty dataframe"): + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.as_dicts())).api_row_df + expected_df = CovidcastRows(rows=[]).api_row_df + assert_frame_equal_no_order(smoothed_df, expected_df, index=["signal", "geo_value", "time_value"]) + + data = CovidcastRows.from_args(time_value=pd.date_range("2021-05-01", "2021-05-13"), value=chain(range(10), [None, 2.0, 1.0])) + with self.subTest("regular window, nan fill"): + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.as_dicts())).api_row_df + expected_df = smooth_df(data.api_row_df, "sig", omit_left_boundary=True) + assert_frame_equal_no_order(smoothed_df, expected_df, index=["signal", "geo_value", "time_value"]) + + with self.subTest("regular window, 0 fill"): + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.as_dicts(), nan_fill_value=0.0)).api_row_df + expected_df = smooth_df(data.api_row_df, "sig", nan_fill_value=0.0, omit_left_boundary=True) + assert_frame_equal_no_order(smoothed_df, expected_df, index=["signal", "geo_value", "time_value"]) + + with self.subTest("regular window, different window length"): + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.as_dicts(), smoother_window_length=8)).api_row_df + expected_df = smooth_df(data.api_row_df, "sig", window_length=8, omit_left_boundary=True) + smoothed_df[["time_value", "value"]] + assert_frame_equal_no_order(smoothed_df, expected_df, index=["signal", "geo_value", "time_value"]) + + def test_generate_diffed_rows(self): + data = CovidcastRows(rows=[]) + with self.subTest("an empty dataframe should return an empty dataframe"): + diffs_df = CovidcastRows.from_records(generate_diffed_rows(data.as_dicts())).api_row_df + expected_df = CovidcastRows(rows=[]).api_row_df + assert_frame_equal_no_order(diffs_df, expected_df, index=["signal", "geo_value", "time_value"]) + + data = CovidcastRows.from_args(time_value=[20210501], value=[1.0]) + with self.subTest("a dataframe with not enough data to make one row should return an empty dataframe"): + diffs_df = CovidcastRows.from_records(generate_diffed_rows(data.as_dicts())).api_row_df + expected_df = diff_df(data.api_row_df, "sig", omit_left_boundary=True) + assert_frame_equal_no_order(diffs_df, expected_df, index=["signal", "geo_value", "time_value"]) + + data = CovidcastRows.from_args(time_value=pd.date_range("2021-05-01", "2021-05-10"), value=chain(range(7), [None, 2.0, 1.0])) + with self.subTest("no fill"): + diffs_df = CovidcastRows.from_records(generate_diffed_rows(data.as_dicts())).api_row_df + expected_df = diff_df(data.api_row_df, "sig", omit_left_boundary=True) + assert_frame_equal_no_order(diffs_df, expected_df, index=["signal", "geo_value", "time_value"]) + + with self.subTest("yes fill"): + diffs_df = CovidcastRows.from_records(generate_diffed_rows(data.as_dicts(), nan_fill_value=2.0)).api_row_df + expected_df = diff_df(data.api_row_df, "sig", nan_fill_value=2.0, omit_left_boundary=True) + assert_frame_equal_no_order(diffs_df, expected_df, index=["signal", "geo_value", "time_value"]) diff --git a/tests/server/test_params.py b/tests/server/test_params.py index 2d22a5d37..5c2fddbcb 100644 --- a/tests/server/test_params.py +++ b/tests/server/test_params.py @@ -3,6 +3,7 @@ # standard library from math import inf import unittest +from unittest.mock import patch # from flask.testing import FlaskClient from delphi.epidata.server._common import app @@ -19,15 +20,19 @@ GeoPair, TimePair, SourceSignalPair, + _combine_source_signal_pairs ) from delphi.epidata.server._exceptions import ( ValidationFailedException, ) +from delphi.epidata.server.endpoints.covidcast_utils.test_utils import DATA_SOURCES_BY_ID, DATA_SIGNALS_BY_KEY # py3tester coverage target __test_target__ = "delphi.epidata.server._params" +@patch("delphi.epidata.server.endpoints.covidcast_utils.model.data_sources_by_id", DATA_SOURCES_BY_ID) +@patch("delphi.epidata.server.endpoints.covidcast_utils.model.data_signals_by_key", DATA_SIGNALS_BY_KEY) class UnitTests(unittest.TestCase): """Basic unit tests.""" @@ -182,8 +187,7 @@ def test_parse_source_signal_arg(self): self.assertEqual( parse_source_signal_arg(), [ - SourceSignalPair("src1", ["sig1"]), - SourceSignalPair("src1", ["sig4"]), + SourceSignalPair("src1", ["sig1", "sig4"]), ], ) with self.subTest("multi list"): @@ -191,17 +195,17 @@ def test_parse_source_signal_arg(self): self.assertEqual( parse_source_signal_arg(), [ - SourceSignalPair("src1", ["sig1", "sig2"]), SourceSignalPair("county", ["sig5", "sig6"]), + SourceSignalPair("src1", ["sig1", "sig2"]), ], ) with self.subTest("hybrid"): - with app.test_request_context("/?signal=src2:*;src1:sig4;src3:sig5,sig6"): + with app.test_request_context("/?signal=src2:*;src1:sig4;src3:sig5,sig6;src1:sig5;src2:sig1"): self.assertEqual( parse_source_signal_arg(), [ + SourceSignalPair("src1", ["sig4", "sig5"]), SourceSignalPair("src2", True), - SourceSignalPair("src1", ["sig4"]), SourceSignalPair("src3", ["sig5", "sig6"]), ], ) @@ -366,3 +370,29 @@ def test_parse_day_arg(self): self.assertRaises(ValidationFailedException, parse_day_arg, "time") with app.test_request_context("/?time=week:20121010"): self.assertRaises(ValidationFailedException, parse_day_arg, "time") + + def test__combine_source_signal_pairs(self): + source_signal_pairs = [ + SourceSignalPair("src1", ["sig1", "sig2"]), + SourceSignalPair("src2", ["sig1"]), + SourceSignalPair("src1", ["sig1", "sig3"]), + SourceSignalPair("src3", ["sig1"]), + SourceSignalPair("src3", ["sig2"]), + SourceSignalPair("src3", ["sig1"]), + SourceSignalPair("src4", ["sig2"]), + SourceSignalPair("src4", True), + ] + expected_source_signal_pairs = [ + SourceSignalPair("src1", ["sig1", "sig2", "sig3"]), + SourceSignalPair("src2", ["sig1"]), + SourceSignalPair("src3", ["sig1", "sig2"]), + SourceSignalPair("src4", True), + ] + combined_pairs = _combine_source_signal_pairs(source_signal_pairs) + for i, x in enumerate(combined_pairs): + if isinstance(x, list): + sorted(x) == expected_source_signal_pairs[i] + if isinstance(x, bool): + x == expected_source_signal_pairs[i] + + assert _combine_source_signal_pairs(source_signal_pairs) == expected_source_signal_pairs diff --git a/tests/server/test_validate.py b/tests/server/test_validate.py index c254950ff..f18a0c2c6 100644 --- a/tests/server/test_validate.py +++ b/tests/server/test_validate.py @@ -1,8 +1,9 @@ """Unit tests for granular sensor authentication in api.php.""" # standard library -import unittest import base64 +import math +import unittest # from flask.testing import FlaskClient from delphi.epidata.server._common import app @@ -14,6 +15,7 @@ extract_strings, extract_integers, extract_integer, + extract_float, extract_date, extract_dates ) @@ -139,6 +141,20 @@ def test_extract_integer(self): with app.test_request_context("/?s=a"): self.assertRaises(ValidationFailedException, lambda: extract_integer("s")) + def test_extract_float(self): + with self.subTest("empty"): + with app.test_request_context("/"): + self.assertIsNone(extract_float("s")) + with self.subTest("single"): + with app.test_request_context("/?s=1.0"): + self.assertEqual(extract_float("s"), 1.0) + with self.subTest("not a number"): + with app.test_request_context("/?s=a"): + self.assertRaises(ValidationFailedException, lambda: extract_float("s")) + with self.subTest("nan"): + with app.test_request_context("/?s=nan"): + self.assertTrue(math.isnan(extract_float("s"))) + def test_extract_integers(self): with self.subTest("empty"): with app.test_request_context("/"): diff --git a/tests/server/utils/test_dates.py b/tests/server/utils/test_dates.py index 5c42e6e16..31d03b368 100644 --- a/tests/server/utils/test_dates.py +++ b/tests/server/utils/test_dates.py @@ -2,7 +2,7 @@ from datetime import date from epiweeks import Week -from delphi.epidata.server.utils.dates import time_value_to_day, day_to_time_value, shift_day_value, time_value_to_iso, days_in_range, weeks_in_range, week_to_time_value, time_value_to_week, time_values_to_ranges +from delphi.epidata.server.utils.dates import time_value_to_day, day_to_time_value, shift_day_value, time_value_to_iso, days_in_range, weeks_in_range, week_to_time_value, time_value_to_week, time_values_to_ranges, iterate_over_range, iterate_over_ints_and_ranges class UnitTests(unittest.TestCase): @@ -59,3 +59,19 @@ def test_time_values_to_ranges(self): self.assertEqual(time_values_to_ranges([20210228, 20210301]), [(20210228, 20210301)]) # this becomes a range because these dates are indeed consecutive # individual weeks become a range (2020 is a rare year with 53 weeks) self.assertEqual(time_values_to_ranges([202051, 202052, 202053, 202101, 202102]), [(202051, 202102)]) + + def test_iterate_over_range(self): + self.assertEqual(list(iterate_over_range(20210801, 20210805)), [20210801, 20210802, 20210803, 20210804]) + self.assertEqual(list(iterate_over_range(20210801, 20210801)), []) + self.assertEqual(list(iterate_over_range(20210801, 20210701)), []) + + def test_iterate_over_ints_and_ranges(self): + assert list(iterate_over_ints_and_ranges([0, (5, 8)], use_dates=False)) == [0, 5, 6, 7, 8] + assert list(iterate_over_ints_and_ranges([(5, 8), (4, 6), (3, 5)], use_dates=False)) == [3, 4, 5, 6, 7, 8] + assert list(iterate_over_ints_and_ranges([(7, 8), (5, 7), (3, 8), 8], use_dates=False)) == [3, 4, 5, 6, 7, 8] + assert list(iterate_over_ints_and_ranges([2, (2, 3)], use_dates=False)) == [2, 3] + assert list(iterate_over_ints_and_ranges([20, 50, 25, (21, 25), 23, 30, 31, (24, 26)], use_dates=False)) == [20, 21, 22, 23, 24, 25, 26, 30, 31, 50] + + assert list(iterate_over_ints_and_ranges([20210817])) == [20210817] + assert list(iterate_over_ints_and_ranges([20210817, (20210810, 20210815)])) == [20210810, 20210811, 20210812, 20210813, 20210814, 20210815, 20210817] + assert list(iterate_over_ints_and_ranges([(20210801, 20210905), (20210815, 20210915)])) == list(iterate_over_range(20210801, 20210916)) # right-exclusive From 2391d10bdeac5561d3dac7185aa7f187d801e421 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Tue, 11 Oct 2022 17:42:49 -0400 Subject: [PATCH 19/23] CI: Build a container image from this branch --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5070eae06..28b23cc40 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -106,7 +106,8 @@ jobs: image: needs: build # only on main and dev branch - if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' + #if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/jit_computations' runs-on: ubuntu-latest steps: From 7b0addbbca69aa9a2893cc339e16861a01a331a9 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Thu, 8 Dec 2022 14:29:59 -0800 Subject: [PATCH 20/23] JIT: Small name improvement in model.py --- src/server/endpoints/covidcast_utils/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index 3d4cb0cb5..530560cf4 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -543,9 +543,9 @@ def _generate_transformed_rows( for key, source_signal_geo_rows in groupby(parsed_rows, group_keyfunc): base_source_name, base_signal_name, _, _ = key # Extract the list of derived signals; if a signal is not in the dictionary, then use the identity map. - derived_signal_transform_map: SourceSignalPair = transform_dict.get(SourceSignalPair(base_source_name, [base_signal_name]), SourceSignalPair(base_source_name, [base_signal_name])) + derived_signals: SourceSignalPair = transform_dict.get(SourceSignalPair(base_source_name, [base_signal_name]), SourceSignalPair(base_source_name, [base_signal_name])) # Create a list of source-signal pairs along with the transformation required for the signal. - signal_names_and_transforms: List[Tuple[Tuple[str, str], Callable]] = [(derived_signal, _get_base_signal_transform((base_source_name, derived_signal))) for derived_signal in derived_signal_transform_map.signal] + signal_names_and_transforms: List[Tuple[Tuple[str, str], Callable]] = [(derived_signal, _get_base_signal_transform((base_source_name, derived_signal))) for derived_signal in derived_signals.signal] # Put the current time series on a contiguous time index. source_signal_geo_rows = _reindex_iterable(source_signal_geo_rows, fill_value=transform_args.get("pad_fill_value")) # Create copies of the iterable, with smart memory usage. From e2a2585e3049036404e7d85463c32227a13bda12 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Thu, 8 Dec 2022 14:41:40 -0800 Subject: [PATCH 21/23] JIT: reduce the iterator stack on non-derived signals --- src/server/endpoints/covidcast_utils/model.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index 530560cf4..c98244b75 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -544,6 +544,10 @@ def _generate_transformed_rows( base_source_name, base_signal_name, _, _ = key # Extract the list of derived signals; if a signal is not in the dictionary, then use the identity map. derived_signals: SourceSignalPair = transform_dict.get(SourceSignalPair(base_source_name, [base_signal_name]), SourceSignalPair(base_source_name, [base_signal_name])) + # Speed up base signals by not transforming them. + if derived_signals.signal == [base_signal_name]: + yield from source_signal_geo_rows + continue # Create a list of source-signal pairs along with the transformation required for the signal. signal_names_and_transforms: List[Tuple[Tuple[str, str], Callable]] = [(derived_signal, _get_base_signal_transform((base_source_name, derived_signal))) for derived_signal in derived_signals.signal] # Put the current time series on a contiguous time index. From 4c023f3710bd7665286ac810bf4b1fe0df82aaa7 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 9 Dec 2022 14:09:46 -0800 Subject: [PATCH 22/23] JIT: rewrite a few iterators and reduce the iterator stack --- .../endpoints/covidcast_utils/smooth_diff.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/server/endpoints/covidcast_utils/smooth_diff.py b/src/server/endpoints/covidcast_utils/smooth_diff.py index 80622da26..5605f3397 100644 --- a/src/server/endpoints/covidcast_utils/smooth_diff.py +++ b/src/server/endpoints/covidcast_utils/smooth_diff.py @@ -1,3 +1,4 @@ +import itertools from enum import Enum from logging import getLogger from numbers import Number @@ -5,11 +6,20 @@ import numpy as np from delphi_utils.nancodes import Nans -from more_itertools import windowed from ...utils.dates import time_value_to_day +def windowed(iterable, n): + it = iter(iterable) + result = tuple(itertools.islice(it, n)) + if len(result) == n: + yield result + for elem in it: + result = result[1:] + (elem,) + yield result + + class SmootherKernelValue(str, Enum): average = "average" @@ -52,10 +62,6 @@ def generate_smoothed_rows( nan_fill_value = np.nan for window in windowed(rows, smoother_window_length): # Iterable[List[Dict]] - # This occurs only if len(rows) < smoother_window_length. - if None in window: - continue - new_value = 0 for row in window: if row.get("value") is not None: @@ -90,10 +96,6 @@ def generate_diffed_rows(rows: Iterable[Dict], nan_fill_value: Number = np.nan, nan_fill_value = np.nan for window in windowed(rows, 2): - # This occurs only if len(rows) < 2. - if None in window: - continue - first_value = window[0]["value"] if window[0].get("value") is not None else nan_fill_value second_value = window[1]["value"] if window[1].get("value") is not None else nan_fill_value new_value = float(round(second_value - first_value, 7)) From 2e09038d29c5c515c661c9808c12a8f992903162 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 9 Dec 2022 14:10:46 -0800 Subject: [PATCH 23/23] JIT: iterator optimizations --- src/server/endpoints/covidcast_utils/model.py | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index 530560cf4..c7e988ec2 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -2,13 +2,12 @@ from datetime import timedelta from enum import Enum from functools import partial -from itertools import groupby, repeat, tee +from itertools import groupby, tee from numbers import Number from typing import Callable, Generator, Iterator, Optional, Dict, List, Set, Tuple, Union from pathlib import Path import re -from more_itertools import interleave_longest, peekable import pandas as pd import numpy as np @@ -348,11 +347,14 @@ def _reindex_iterable(iterator: Iterator[Dict], fill_value: Optional[Number] = N The min and max time_values are determined from the first and last rows of the iterator. The fill_value is used to fill in gaps in the time index. """ - _iterator = peekable(iterator) + # Since we're looking ahead, we need to keep a buffer of the last item. + peek_memory = [] # If the iterator is empty, we halt immediately. + iterator = iter(iterator) try: - first_item = _iterator.peek() + first_item = next(iterator) + peek_memory.append(first_item) except StopIteration: return @@ -374,15 +376,21 @@ def _reindex_iterable(iterator: Iterator[Dict], fill_value: Optional[Number] = N # Non-trivial operations otherwise. while True: try: - # This will stay the same until the peeked element is consumed. - new_item = _iterator.peek() + if peek_memory: + new_item = peek_memory.pop() + else: + new_item = next(iterator) except StopIteration: return if expected_time_value == new_item.get("time_value"): - # Get the value we just peeked. - yield next(_iterator) + # Return the row we just peeked. + yield new_item else: + # We've found a gap in the time index. + # Put the new item back in the buffer. + peek_memory.append(new_item) + # Return a default row instead. # Copy to avoid Python by-reference memory issues. default_item = _default_item.copy() @@ -551,12 +559,18 @@ def _generate_transformed_rows( # Create copies of the iterable, with smart memory usage. source_signal_geo_rows_copies: Iterator[Iterator[Dict]] = tee(source_signal_geo_rows, len(signal_names_and_transforms)) # Create a list of transformed group iterables, remembering their derived name as needed. - transformed_signals_iterator: Iterator[Tuple[str, Iterator[Dict]]] = (zip(repeat(derived_signal), transform(rows, **transform_args)) for (derived_signal, transform), rows in zip(signal_names_and_transforms, source_signal_geo_rows_copies)) + transformed_signals_iterators: List[Tuple[str, Iterator[Dict]]] = [(derived_signal, transform(rows, **transform_args)) for (derived_signal, transform), rows in zip(signal_names_and_transforms, source_signal_geo_rows_copies)] + # Traverse through the transformed iterables in an interleaved fashion, which makes sure that only a small window # of the original iterable (group) is stored in memory. - for derived_signal_name, row in interleave_longest(*transformed_signals_iterator): - row["signal"] = derived_signal_name - yield row + while transformed_signals_iterators: + for derived_signal_name, transformed_signal_iterator in transformed_signals_iterators: + try: + row = next(transformed_signal_iterator) + row["signal"] = derived_signal_name + yield row + except StopIteration: + transformed_signals_iterators.remove((derived_signal_name, transformed_signal_iterator)) def get_basename_signal_and_jit_generator(source_signal_pairs: List[SourceSignalPair], transform_args: Optional[Dict[str, Union[str, int]]] = None) -> Tuple[List[SourceSignalPair], Generator]: