diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 69d8ad049..b2688441b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.1.5 +current_version = 4.1.6 commit = False tag = False diff --git a/dev/local/setup.cfg b/dev/local/setup.cfg index 27a715278..f435a1136 100644 --- a/dev/local/setup.cfg +++ b/dev/local/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = Delphi Development -version = 4.1.5 +version = 4.1.6 [options] packages = diff --git a/requirements.api.txt b/requirements.api.txt index e9c1418df..10fed2eec 100644 --- a/requirements.api.txt +++ b/requirements.api.txt @@ -13,7 +13,7 @@ python-dotenv==0.15.0 pyyaml redis==3.5.3 requests==2.31.0 -scipy==1.6.2 +scipy==1.10.0 SQLAlchemy==1.4.40 structlog==22.1.0 tenacity==7.0.0 diff --git a/src/acquisition/covidcast/test_utils.py b/src/acquisition/covidcast/test_utils.py index 6e77aba22..5a978f8cd 100644 --- a/src/acquisition/covidcast/test_utils.py +++ b/src/acquisition/covidcast/test_utils.py @@ -4,17 +4,19 @@ import unittest import pandas as pd +from redis import Redis from delphi_utils import Nans from delphi.epidata.common.covidcast_row import CovidcastRow from delphi.epidata.acquisition.covidcast.database import Database +from delphi.epidata.server._config import REDIS_HOST, REDIS_PASSWORD from delphi.epidata.server.utils.dates import day_to_time_value, time_value_to_day import delphi.operations.secrets as secrets # all the Nans we use here are just one value, so this is a shortcut to it: nmv = Nans.NOT_MISSING.value -# TODO replace these real geo_values with fake values, and use patch and mock to mock the return values of +# TODO replace these real geo_values with fake values, and use patch and mock to mock the return values of # delphi_utils.geomap.GeoMapper().get_geo_values(geo_type) in parse_geo_sets() of _params.py FIPS = ['04019', '19143', '29063', '36083'] # Example list of valid FIPS codes as strings @@ -166,6 +168,12 @@ def setUp(self): self.localSetUp() self._db._connection.commit() + # clear all rate-limiting info from redis + r = Redis(host=REDIS_HOST, password=REDIS_PASSWORD) + for k in r.keys("LIMITER/*"): + r.delete(k) + + def tearDown(self): # close and destroy conenction to the database self.localTearDown() @@ -173,12 +181,12 @@ def tearDown(self): del self._db def localSetUp(self): - # stub; override in subclasses to perform custom setup. + # stub; override in subclasses to perform custom setup. # runs after tables have been truncated but before database changes have been committed pass def localTearDown(self): - # stub; override in subclasses to perform custom teardown. + # stub; override in subclasses to perform custom teardown. # runs after database changes have been committed pass @@ -198,4 +206,4 @@ def params_from_row(self, row: CovidcastTestRow, **kwargs): 'geo_value': row.geo_value, } ret.update(kwargs) - return ret \ No newline at end of file + return ret diff --git a/src/client/delphi_epidata.R b/src/client/delphi_epidata.R index 7a161656f..ee2dcda09 100644 --- a/src/client/delphi_epidata.R +++ b/src/client/delphi_epidata.R @@ -15,7 +15,7 @@ Epidata <- (function() { # API base url BASE_URL <- getOption('epidata.url', default = 'https://api.delphi.cmu.edu/epidata/') - client_version <- '4.1.5' + client_version <- '4.1.6' auth <- getOption("epidata.auth", default = NA) diff --git a/src/client/delphi_epidata.js b/src/client/delphi_epidata.js index 413b93207..8e1023118 100644 --- a/src/client/delphi_epidata.js +++ b/src/client/delphi_epidata.js @@ -22,7 +22,7 @@ } })(this, function (exports, fetchImpl, jQuery) { const BASE_URL = "https://api.delphi.cmu.edu/epidata/"; - const client_version = "4.1.5"; + const client_version = "4.1.6"; // Helper function to cast values and/or ranges to strings function _listitem(value) { diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index 654eba74c..fe8dbe51d 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -20,684 +20,731 @@ # preference, even if you've installed it and then use this script independently # by accident. try: - _version = get_distribution('delphi-epidata').version + _version = get_distribution("delphi-epidata").version except DistributionNotFound: - _version = "0.script" + _version = "0.script" -_HEADERS = { - "user-agent": "delphi_epidata/" + _version + " (Python)" -} +_HEADERS = {"user-agent": "delphi_epidata/" + _version + " (Python)"} + + +class EpidataException(Exception): + pass + + +class EpidataBadRequestException(EpidataException): + pass + + +REGIONS_EPIWEEKS_REQUIRED = "`regions` and `epiweeks` are both required" +ISSUES_LAG_EXCLUSIVE = "`issues` and `lag` are mutually exclusive" +LOCATIONS_EPIWEEKS_REQUIRED = "`locations` and `epiweeks` are both required" # Because the API is stateless, the Epidata class only contains static methods class Epidata: - """An interface to DELPHI's Epidata API.""" - - # API base url - BASE_URL = 'https://api.delphi.cmu.edu/epidata/api.php' - auth = None - - client_version = _version - - # Helper function to cast values and/or ranges to strings - @staticmethod - def _listitem(value): - """Cast values and/or range to a string.""" - if isinstance(value, dict) and 'from' in value and 'to' in value: - return str(value['from']) + '-' + str(value['to']) - else: - return str(value) - - # Helper function to build a list of values and/or ranges - @staticmethod - def _list(values): - """Turn a list/tuple of values/ranges into a comma-separated string.""" - if not isinstance(values, (list, tuple)): - values = [values] - return ','.join([Epidata._listitem(value) for value in values]) - - @staticmethod - @retry(reraise=True, stop=stop_after_attempt(2)) - def _request_with_retry(params): - """Make request with a retry if an exception is thrown.""" - req = requests.get(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) - if req.status_code == 414: - req = requests.post(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) - # handle 401 and 429 - req.raise_for_status() - return req - - @staticmethod - def _request(params): - """Request and parse epidata. - - We default to GET since it has better caching and logging - capabilities, but fall back to POST if the request is too - long and returns a 414. - """ - try: - result = Epidata._request_with_retry(params) - except Exception as e: - return {'result': 0, 'message': 'error: ' + str(e)} - if params is not None and "format" in params and params["format"]=="csv": - return result.text - else: - try: - return result.json() - except requests.exceptions.JSONDecodeError: - return {'result': 0, 'message': 'error decoding json: ' + result.text} - - # Raise an Exception on error, otherwise return epidata - @staticmethod - def check(resp): - """Raise an Exception on error, otherwise return epidata.""" - if resp['result'] != 1: - msg, code = resp['message'], resp['result'] - raise Exception('Error fetching epidata: %s. (result=%d)' % (msg, code)) - return resp['epidata'] - - # Build a `range` object (ex: dates/epiweeks) - @staticmethod - def range(from_, to_): - """Build a `range` object (ex: dates/epiweeks).""" - if to_ <= from_: - from_, to_ = to_, from_ - return {'from': from_, 'to': to_} - - # Fetch FluView data - @staticmethod - def fluview(regions, epiweeks, issues=None, lag=None, auth=None): - """Fetch FluView data.""" - # Check parameters - if regions is None or epiweeks is None: - raise Exception('`regions` and `epiweeks` are both required') - if issues is not None and lag is not None: - raise Exception('`issues` and `lag` are mutually exclusive') - # Set up request - params = { - 'endpoint': 'fluview', - 'regions': Epidata._list(regions), - 'epiweeks': Epidata._list(epiweeks), - } - if issues is not None: - params['issues'] = Epidata._list(issues) - if lag is not None: - params['lag'] = lag - if auth is not None: - params['auth'] = auth - # Make the API call - return Epidata._request(params) - - # Fetch FluView metadata - @staticmethod - def fluview_meta(): - """Fetch FluView metadata.""" - # Set up request - params = { - 'endpoint': 'fluview_meta', - } - # Make the API call - return Epidata._request(params) - - # Fetch FluView clinical data - @staticmethod - def fluview_clinical(regions, epiweeks, issues=None, lag=None): - """Fetch FluView clinical data.""" - # Check parameters - if regions is None or epiweeks is None: - raise Exception('`regions` and `epiweeks` are both required') - if issues is not None and lag is not None: - raise Exception('`issues` and `lag` are mutually exclusive') - # Set up request - params = { - 'endpoint': 'fluview_clinical', - 'regions': Epidata._list(regions), - 'epiweeks': Epidata._list(epiweeks), - } - if issues is not None: - params['issues'] = Epidata._list(issues) - if lag is not None: - params['lag'] = lag - # Make the API call - return Epidata._request(params) - - # Fetch FluSurv data - @staticmethod - def flusurv(locations, epiweeks, issues=None, lag=None): - """Fetch FluSurv data.""" - # Check parameters - if locations is None or epiweeks is None: - raise Exception('`locations` and `epiweeks` are both required') - if issues is not None and lag is not None: - raise Exception('`issues` and `lag` are mutually exclusive') - # Set up request - params = { - 'endpoint': 'flusurv', - 'locations': Epidata._list(locations), - 'epiweeks': Epidata._list(epiweeks), - } - if issues is not None: - params['issues'] = Epidata._list(issues) - if lag is not None: - params['lag'] = lag - # Make the API call - return Epidata._request(params) - - # Fetch PAHO Dengue data - @staticmethod - def paho_dengue(regions, epiweeks, issues=None, lag=None): - """Fetch PAHO Dengue data.""" - # Check parameters - if regions is None or epiweeks is None: - raise Exception('`regions` and `epiweeks` are both required') - if issues is not None and lag is not None: - raise Exception('`issues` and `lag` are mutually exclusive') - # Set up request - params = { - 'endpoint': 'paho_dengue', - 'regions': Epidata._list(regions), - 'epiweeks': Epidata._list(epiweeks), - } - if issues is not None: - params['issues'] = Epidata._list(issues) - if lag is not None: - params['lag'] = lag - # Make the API call - return Epidata._request(params) - - # Fetch ECDC ILI data - @staticmethod - def ecdc_ili(regions, epiweeks, issues=None, lag=None): - """Fetch ECDC ILI data.""" - # Check parameters - if regions is None or epiweeks is None: - raise Exception('`regions` and `epiweeks` are both required') - if issues is not None and lag is not None: - raise Exception('`issues` and `lag` are mutually exclusive') - # Set up request - params = { - 'endpoint': 'ecdc_ili', - 'regions': Epidata._list(regions), - 'epiweeks': Epidata._list(epiweeks), - } - if issues is not None: - params['issues'] = Epidata._list(issues) - if lag is not None: - params['lag'] = lag - # Make the API call - return Epidata._request(params) - - # Fetch KCDC ILI data - @staticmethod - def kcdc_ili(regions, epiweeks, issues=None, lag=None): - """Fetch KCDC ILI data.""" - # Check parameters - if regions is None or epiweeks is None: - raise Exception('`regions` and `epiweeks` are both required') - if issues is not None and lag is not None: - raise Exception('`issues` and `lag` are mutually exclusive') - # Set up request - params = { - 'endpoint': 'kcdc_ili', - 'regions': Epidata._list(regions), - 'epiweeks': Epidata._list(epiweeks), - } - if issues is not None: - params['issues'] = Epidata._list(issues) - if lag is not None: - params['lag'] = lag - # Make the API call - return Epidata._request(params) - - # Fetch Google Flu Trends data - @staticmethod - def gft(locations, epiweeks): - """Fetch Google Flu Trends data.""" - # Check parameters - if locations is None or epiweeks is None: - raise Exception('`locations` and `epiweeks` are both required') - # Set up request - params = { - 'endpoint': 'gft', - 'locations': Epidata._list(locations), - 'epiweeks': Epidata._list(epiweeks), - } - # Make the API call - return Epidata._request(params) - - # Fetch Google Health Trends data - @staticmethod - def ght(auth, locations, epiweeks, query): - """Fetch Google Health Trends data.""" - # Check parameters - if auth is None or locations is None or epiweeks is None or query is None: - raise Exception('`auth`, `locations`, `epiweeks`, and `query` are all required') - # Set up request - params = { - 'endpoint': 'ght', - 'auth': auth, - 'locations': Epidata._list(locations), - 'epiweeks': Epidata._list(epiweeks), - 'query': query, - } - # Make the API call - return Epidata._request(params) - - # Fetch HealthTweets data - @staticmethod - def twitter(auth, locations, dates=None, epiweeks=None): - """Fetch HealthTweets data.""" - # Check parameters - if auth is None or locations is None: - raise Exception('`auth` and `locations` are both required') - if not ((dates is None) ^ (epiweeks is None)): - raise Exception('exactly one of `dates` and `epiweeks` is required') - # Set up request - params = { - 'endpoint': 'twitter', - 'auth': auth, - 'locations': Epidata._list(locations), - } - if dates is not None: - params['dates'] = Epidata._list(dates) - if epiweeks is not None: - params['epiweeks'] = Epidata._list(epiweeks) - # Make the API call - return Epidata._request(params) - - # Fetch Wikipedia access data - @staticmethod - def wiki(articles, dates=None, epiweeks=None, hours=None, language='en'): - """Fetch Wikipedia access data.""" - # Check parameters - if articles is None: - raise Exception('`articles` is required') - if not ((dates is None) ^ (epiweeks is None)): - raise Exception('exactly one of `dates` and `epiweeks` is required') - # Set up request - params = { - 'endpoint': 'wiki', - 'articles': Epidata._list(articles), - 'language': language, - } - if dates is not None: - params['dates'] = Epidata._list(dates) - if epiweeks is not None: - params['epiweeks'] = Epidata._list(epiweeks) - if hours is not None: - params['hours'] = Epidata._list(hours) - # Make the API call - return Epidata._request(params) - - # Fetch CDC page hits - @staticmethod - def cdc(auth, epiweeks, locations): - """Fetch CDC page hits.""" - # Check parameters - if auth is None or epiweeks is None or locations is None: - raise Exception('`auth`, `epiweeks`, and `locations` are all required') - # Set up request - params = { - 'endpoint': 'cdc', - 'auth': auth, - 'epiweeks': Epidata._list(epiweeks), - 'locations': Epidata._list(locations), - } - # Make the API call - return Epidata._request(params) - - # Fetch Quidel data - @staticmethod - def quidel(auth, epiweeks, locations): - """Fetch Quidel data.""" - # Check parameters - if auth is None or epiweeks is None or locations is None: - raise Exception('`auth`, `epiweeks`, and `locations` are all required') - # Set up request - params = { - 'endpoint': 'quidel', - 'auth': auth, - 'epiweeks': Epidata._list(epiweeks), - 'locations': Epidata._list(locations), - } - # Make the API call - return Epidata._request(params) - - # Fetch NoroSTAT data (point data, no min/max) - @staticmethod - def norostat(auth, location, epiweeks): - """Fetch NoroSTAT data (point data, no min/max).""" - # Check parameters - if auth is None or location is None or epiweeks is None: - raise Exception('`auth`, `location`, and `epiweeks` are all required') - # Set up request - params = { - 'endpoint': 'norostat', - 'auth': auth, - 'location': location, - 'epiweeks': Epidata._list(epiweeks), - } - # Make the API call - return Epidata._request(params) - - # Fetch NoroSTAT metadata - @staticmethod - def meta_norostat(auth): - """Fetch NoroSTAT metadata.""" - # Check parameters - if auth is None: - raise Exception('`auth` is required') - # Set up request - params = { - 'endpoint': 'meta_norostat', - 'auth': auth, - } - # Make the API call - return Epidata._request(params) - - # Fetch NIDSS flu data - @staticmethod - def nidss_flu(regions, epiweeks, issues=None, lag=None): - """Fetch NIDSS flu data.""" - # Check parameters - if regions is None or epiweeks is None: - raise Exception('`regions` and `epiweeks` are both required') - if issues is not None and lag is not None: - raise Exception('`issues` and `lag` are mutually exclusive') - # Set up request - params = { - 'endpoint': 'nidss_flu', - 'regions': Epidata._list(regions), - 'epiweeks': Epidata._list(epiweeks), - } - if issues is not None: - params['issues'] = Epidata._list(issues) - if lag is not None: - params['lag'] = lag - # Make the API call - return Epidata._request(params) - - # Fetch NIDSS dengue data - @staticmethod - def nidss_dengue(locations, epiweeks): - """Fetch NIDSS dengue data.""" - # Check parameters - if locations is None or epiweeks is None: - raise Exception('`locations` and `epiweeks` are both required') - # Set up request - params = { - 'endpoint': 'nidss_dengue', - 'locations': Epidata._list(locations), - 'epiweeks': Epidata._list(epiweeks), - } - # Make the API call - return Epidata._request(params) - - # Fetch Delphi's forecast - @staticmethod - def delphi(system, epiweek): - """Fetch Delphi's forecast.""" - # Check parameters - if system is None or epiweek is None: - raise Exception('`system` and `epiweek` are both required') - # Set up request - params = { - 'endpoint': 'delphi', - 'system': system, - 'epiweek': epiweek, - } - # Make the API call - return Epidata._request(params) - - # Fetch Delphi's digital surveillance sensors - @staticmethod - def sensors(auth, names, locations, epiweeks): - """Fetch Delphi's digital surveillance sensors.""" - # Check parameters - if names is None or locations is None or epiweeks is None: - raise Exception('`names`, `locations`, and `epiweeks` are all required') - # Set up request - params = { - 'endpoint': 'sensors', - 'names': Epidata._list(names), - 'locations': Epidata._list(locations), - 'epiweeks': Epidata._list(epiweeks), - } - if auth is not None: - params['auth'] = auth - # Make the API call - return Epidata._request(params) - - # Fetch Delphi's dengue digital surveillance sensors - @staticmethod - def dengue_sensors(auth, names, locations, epiweeks): - """Fetch Delphi's digital surveillance sensors.""" - # Check parameters - if auth is None or names is None or locations is None or epiweeks is None: - raise Exception('`auth`, `names`, `locations`, and `epiweeks` are all required') - # Set up request - params = { - 'endpoint': 'dengue_sensors', - 'auth': auth, - 'names': Epidata._list(names), - 'locations': Epidata._list(locations), - 'epiweeks': Epidata._list(epiweeks), - } - # Make the API call - return Epidata._request(params) - - # Fetch Delphi's wILI nowcast - @staticmethod - def nowcast(locations, epiweeks): - """Fetch Delphi's wILI nowcast.""" - # Check parameters - if locations is None or epiweeks is None: - raise Exception('`locations` and `epiweeks` are both required') - # Set up request - params = { - 'endpoint': 'nowcast', - 'locations': Epidata._list(locations), - 'epiweeks': Epidata._list(epiweeks), - } - # Make the API call - return Epidata._request(params) - - # Fetch Delphi's dengue nowcast - @staticmethod - def dengue_nowcast(locations, epiweeks): - """Fetch Delphi's dengue nowcast.""" - # Check parameters - if locations is None or epiweeks is None: - raise Exception('`locations` and `epiweeks` are both required') - # Set up request - params = { - 'endpoint': 'dengue_nowcast', - 'locations': Epidata._list(locations), - 'epiweeks': Epidata._list(epiweeks), - } - # Make the API call - return Epidata._request(params) - - # Fetch API metadata - @staticmethod - def meta(): - """Fetch API metadata.""" - return Epidata._request({'endpoint': 'meta'}) - - # Fetch Delphi's COVID-19 Surveillance Streams - @staticmethod - def covidcast( - data_source, signals, time_type, geo_type, - time_values, geo_value, as_of=None, issues=None, lag=None, **kwargs): - """Fetch Delphi's COVID-19 Surveillance Streams""" - # also support old parameter name - if signals is None and 'signal' in kwargs: - signals=kwargs['signal'] - # Check parameters - if data_source is None or signals is None or time_type is None or geo_type is None or time_values is None or geo_value is None: - raise Exception('`data_source`, `signals`, `time_type`, `geo_type`, `time_values`, and `geo_value` are all required') - if issues is not None and lag is not None: - raise Exception('`issues` and `lag` are mutually exclusive') - # Set up request - params = { - 'endpoint': 'covidcast', - 'data_source': data_source, - 'signals': Epidata._list(signals), - 'time_type': time_type, - 'geo_type': geo_type, - 'time_values': Epidata._list(time_values) - } - - if isinstance(geo_value, (list, tuple)): - params['geo_values'] = ','.join(geo_value) - else: - params['geo_value'] = geo_value - if as_of is not None: - params['as_of'] = as_of - if issues is not None: - params['issues'] = Epidata._list(issues) - if lag is not None: - params['lag'] = lag - - if 'format' in kwargs: - params['format'] = kwargs['format'] - - if 'fields' in kwargs: - params['fields'] = kwargs['fields'] - - # Make the API call - return Epidata._request(params) - - # Fetch Delphi's COVID-19 Surveillance Streams metadata - @staticmethod - def covidcast_meta(): - """Fetch Delphi's COVID-19 Surveillance Streams metadata""" - return Epidata._request({'endpoint': 'covidcast_meta'}) - - # Fetch COVID hospitalization data - @staticmethod - def covid_hosp(states, dates, issues=None, as_of=None): - """Fetch COVID hospitalization data.""" - # Check parameters - if states is None or dates is None: - raise Exception('`states` and `dates` are both required') - # Set up request - params = { - 'endpoint': 'covid_hosp', - 'states': Epidata._list(states), - 'dates': Epidata._list(dates), - } - if issues is not None: - params['issues'] = Epidata._list(issues) - if as_of is not None: - params['as_of'] = as_of - # Make the API call - return Epidata._request(params) - - # Fetch COVID hospitalization data for specific facilities - @staticmethod - def covid_hosp_facility( - hospital_pks, collection_weeks, publication_dates=None): - """Fetch COVID hospitalization data for specific facilities.""" - # Check parameters - if hospital_pks is None or collection_weeks is None: - raise Exception('`hospital_pks` and `collection_weeks` are both required') - # Set up request - params = { - 'source': 'covid_hosp_facility', - 'hospital_pks': Epidata._list(hospital_pks), - 'collection_weeks': Epidata._list(collection_weeks), - } - if publication_dates is not None: - params['publication_dates'] = Epidata._list(publication_dates) - # Make the API call - return Epidata._request(params) - - # Lookup COVID hospitalization facility identifiers - @staticmethod - def covid_hosp_facility_lookup( - state=None, ccn=None, city=None, zip=None, fips_code=None): - """Lookup COVID hospitalization facility identifiers.""" - # Set up request - params = {'source': 'covid_hosp_facility_lookup'} - if state is not None: - params['state'] = state - elif ccn is not None: - params['ccn'] = ccn - elif city is not None: - params['city'] = city - elif zip is not None: - params['zip'] = zip - elif fips_code is not None: - params['fips_code'] = fips_code - else: - raise Exception('one of `state`, `ccn`, `city`, `zip`, or `fips_code` is required') - # Make the API call - return Epidata._request(params) - - # Fetch Delphi's COVID-19 Nowcast sensors - @staticmethod - def covidcast_nowcast( - data_source, signals, sensor_names, time_type, geo_type, - time_values, geo_value, as_of=None, issues=None, lag=None, **kwargs): - """Fetch Delphi's COVID-19 Nowcast sensors""" - # Check parameters - if data_source is None or signals is None or time_type is None or geo_type is None or time_values is None or geo_value is None or sensor_names is None: - raise Exception('`data_source`, `signals`, `sensor_names`, `time_type`, `geo_type`, `time_values`, and `geo_value` are all required') - if issues is not None and lag is not None: - raise Exception('`issues` and `lag` are mutually exclusive') - # Set up request - params = { - 'source': 'covidcast_nowcast', - 'data_source': data_source, - 'signals': Epidata._list(signals), - 'sensor_names': Epidata._list(sensor_names), - 'time_type': time_type, - 'geo_type': geo_type, - 'time_values': Epidata._list(time_values) - } - - if isinstance(geo_value, (list, tuple)): - params['geo_values'] = ','.join(geo_value) - else: - params['geo_value'] = geo_value - if as_of is not None: - params['as_of'] = as_of - if issues is not None: - params['issues'] = Epidata._list(issues) - if lag is not None: - params['lag'] = lag - - if 'format' in kwargs: - params['format'] = kwargs['format'] - - # Make the API call - return Epidata._request(params) - - @staticmethod - def async_epidata(param_list, batch_size=50): - """Make asynchronous Epidata calls for a list of parameters.""" - async def async_get(params, session): - """Helper function to make Epidata GET requests.""" - async with session.get(Epidata.BASE_URL, params=params) as response: - response.raise_for_status() - return await response.json(), params - - async def async_make_calls(param_combos): - """Helper function to asynchronously make and aggregate Epidata GET requests.""" - tasks = [] - connector = TCPConnector(limit=batch_size) - if isinstance(Epidata.auth, tuple): - auth = BasicAuth(login=Epidata.auth[0], password=Epidata.auth[1], encoding='utf-8') - else: - auth = Epidata.auth - async with ClientSession(connector=connector, headers=_HEADERS, auth=auth) as session: - for param in param_combos: - task = asyncio.ensure_future(async_get(param, session)) - tasks.append(task) - responses = await asyncio.gather(*tasks) + """An interface to DELPHI's Epidata API.""" + + # API base url + BASE_URL = "https://api.delphi.cmu.edu/epidata/api.php" + auth = None + + client_version = _version + + # Helper function to cast values and/or ranges to strings + @staticmethod + def _listitem(value): + """Cast values and/or range to a string.""" + if isinstance(value, dict) and "from" in value and "to" in value: + return str(value["from"]) + "-" + str(value["to"]) + else: + return str(value) + + # Helper function to build a list of values and/or ranges + @staticmethod + def _list(values): + """Turn a list/tuple of values/ranges into a comma-separated string.""" + if not isinstance(values, (list, tuple)): + values = [values] + return ",".join([Epidata._listitem(value) for value in values]) + + @staticmethod + @retry(reraise=True, stop=stop_after_attempt(2)) + def _request_with_retry(params): + """Make request with a retry if an exception is thrown.""" + req = requests.get(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) + if req.status_code == 414: + req = requests.post(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) + # handle 401 and 429 + req.raise_for_status() + return req + + @staticmethod + def _request(params): + """Request and parse epidata. + + We default to GET since it has better caching and logging + capabilities, but fall back to POST if the request is too + long and returns a 414. + """ + try: + result = Epidata._request_with_retry(params) + except Exception as e: + return {"result": 0, "message": "error: " + str(e)} + if params is not None and "format" in params and params["format"] == "csv": + return result.text + else: + try: + return result.json() + except requests.exceptions.JSONDecodeError: + return {"result": 0, "message": "error decoding json: " + result.text} + + # Raise an Exception on error, otherwise return epidata + @staticmethod + def check(resp): + """Raise an Exception on error, otherwise return epidata.""" + if resp["result"] != 1: + msg, code = resp["message"], resp["result"] + raise EpidataException(f"Error fetching epidata: {msg}. (result={int(code)})") + return resp["epidata"] + + # Build a `range` object (ex: dates/epiweeks) + @staticmethod + def range(from_, to_): + """Build a `range` object (ex: dates/epiweeks).""" + if to_ <= from_: + from_, to_ = to_, from_ + return {"from": from_, "to": to_} + + # Fetch FluView data + @staticmethod + def fluview(regions, epiweeks, issues=None, lag=None, auth=None): + """Fetch FluView data.""" + # Check parameters + if regions is None or epiweeks is None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + if issues is not None and lag is not None: + raise EpidataBadRequestException(ISSUES_LAG_EXCLUSIVE) + # Set up request + params = { + "endpoint": "fluview", + "regions": Epidata._list(regions), + "epiweeks": Epidata._list(epiweeks), + } + if issues is not None: + params["issues"] = Epidata._list(issues) + if lag is not None: + params["lag"] = lag + if auth is not None: + params["auth"] = auth + # Make the API call + return Epidata._request(params) + + # Fetch FluView metadata + @staticmethod + def fluview_meta(): + """Fetch FluView metadata.""" + # Set up request + params = { + "endpoint": "fluview_meta", + } + # Make the API call + return Epidata._request(params) + + # Fetch FluView clinical data + @staticmethod + def fluview_clinical(regions, epiweeks, issues=None, lag=None): + """Fetch FluView clinical data.""" + # Check parameters + if regions is None or epiweeks is None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + if issues is not None and lag is not None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "fluview_clinical", + "regions": Epidata._list(regions), + "epiweeks": Epidata._list(epiweeks), + } + if issues is not None: + params["issues"] = Epidata._list(issues) + if lag is not None: + params["lag"] = lag + # Make the API call + return Epidata._request(params) + + # Fetch FluSurv data + @staticmethod + def flusurv(locations, epiweeks, issues=None, lag=None): + """Fetch FluSurv data.""" + # Check parameters + if locations is None or epiweeks is None: + raise EpidataBadRequestException(LOCATIONS_EPIWEEKS_REQUIRED) + if issues is not None and lag is not None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "flusurv", + "locations": Epidata._list(locations), + "epiweeks": Epidata._list(epiweeks), + } + if issues is not None: + params["issues"] = Epidata._list(issues) + if lag is not None: + params["lag"] = lag + # Make the API call + return Epidata._request(params) + + # Fetch PAHO Dengue data + @staticmethod + def paho_dengue(regions, epiweeks, issues=None, lag=None): + """Fetch PAHO Dengue data.""" + # Check parameters + if regions is None or epiweeks is None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + if issues is not None and lag is not None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "paho_dengue", + "regions": Epidata._list(regions), + "epiweeks": Epidata._list(epiweeks), + } + if issues is not None: + params["issues"] = Epidata._list(issues) + if lag is not None: + params["lag"] = lag + # Make the API call + return Epidata._request(params) + + # Fetch ECDC ILI data + @staticmethod + def ecdc_ili(regions, epiweeks, issues=None, lag=None): + """Fetch ECDC ILI data.""" + # Check parameters + if regions is None or epiweeks is None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + if issues is not None and lag is not None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "ecdc_ili", + "regions": Epidata._list(regions), + "epiweeks": Epidata._list(epiweeks), + } + if issues is not None: + params["issues"] = Epidata._list(issues) + if lag is not None: + params["lag"] = lag + # Make the API call + return Epidata._request(params) + + # Fetch KCDC ILI data + @staticmethod + def kcdc_ili(regions, epiweeks, issues=None, lag=None): + """Fetch KCDC ILI data.""" + # Check parameters + if regions is None or epiweeks is None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + if issues is not None and lag is not None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "kcdc_ili", + "regions": Epidata._list(regions), + "epiweeks": Epidata._list(epiweeks), + } + if issues is not None: + params["issues"] = Epidata._list(issues) + if lag is not None: + params["lag"] = lag + # Make the API call + return Epidata._request(params) + + # Fetch Google Flu Trends data + @staticmethod + def gft(locations, epiweeks): + """Fetch Google Flu Trends data.""" + # Check parameters + if locations is None or epiweeks is None: + raise EpidataBadRequestException(LOCATIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "gft", + "locations": Epidata._list(locations), + "epiweeks": Epidata._list(epiweeks), + } + # Make the API call + return Epidata._request(params) + + # Fetch Google Health Trends data + @staticmethod + def ght(auth, locations, epiweeks, query): + """Fetch Google Health Trends data.""" + # Check parameters + if auth is None or locations is None or epiweeks is None or query is None: + raise EpidataBadRequestException( + "`auth`, `locations`, `epiweeks`, and `query` are all required" + ) + # Set up request + params = { + "endpoint": "ght", + "auth": auth, + "locations": Epidata._list(locations), + "epiweeks": Epidata._list(epiweeks), + "query": query, + } + # Make the API call + return Epidata._request(params) + + # Fetch HealthTweets data + @staticmethod + def twitter(auth, locations, dates=None, epiweeks=None): + """Fetch HealthTweets data.""" + # Check parameters + if auth is None or locations is None: + raise EpidataBadRequestException("`auth` and `locations` are both required") + if not ((dates is None) ^ (epiweeks is None)): + raise EpidataBadRequestException("exactly one of `dates` and `epiweeks` is required") + # Set up request + params = { + "endpoint": "twitter", + "auth": auth, + "locations": Epidata._list(locations), + } + if dates is not None: + params["dates"] = Epidata._list(dates) + if epiweeks is not None: + params["epiweeks"] = Epidata._list(epiweeks) + # Make the API call + return Epidata._request(params) + + # Fetch Wikipedia access data + @staticmethod + def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): + """Fetch Wikipedia access data.""" + # Check parameters + if articles is None: + raise EpidataBadRequestException("`articles` is required") + if not ((dates is None) ^ (epiweeks is None)): + raise EpidataBadRequestException("exactly one of `dates` and `epiweeks` is required") + # Set up request + params = { + "endpoint": "wiki", + "articles": Epidata._list(articles), + "language": language, + } + if dates is not None: + params["dates"] = Epidata._list(dates) + if epiweeks is not None: + params["epiweeks"] = Epidata._list(epiweeks) + if hours is not None: + params["hours"] = Epidata._list(hours) + # Make the API call + return Epidata._request(params) + + # Fetch CDC page hits + @staticmethod + def cdc(auth, epiweeks, locations): + """Fetch CDC page hits.""" + # Check parameters + if auth is None or epiweeks is None or locations is None: + raise EpidataBadRequestException("`auth`, `epiweeks`, and `locations` are all required") + # Set up request + params = { + "endpoint": "cdc", + "auth": auth, + "epiweeks": Epidata._list(epiweeks), + "locations": Epidata._list(locations), + } + # Make the API call + return Epidata._request(params) + + # Fetch Quidel data + @staticmethod + def quidel(auth, epiweeks, locations): + """Fetch Quidel data.""" + # Check parameters + if auth is None or epiweeks is None or locations is None: + raise EpidataBadRequestException("`auth`, `epiweeks`, and `locations` are all required") + # Set up request + params = { + "endpoint": "quidel", + "auth": auth, + "epiweeks": Epidata._list(epiweeks), + "locations": Epidata._list(locations), + } + # Make the API call + return Epidata._request(params) + + # Fetch NoroSTAT data (point data, no min/max) + @staticmethod + def norostat(auth, location, epiweeks): + """Fetch NoroSTAT data (point data, no min/max).""" + # Check parameters + if auth is None or location is None or epiweeks is None: + raise EpidataBadRequestException("`auth`, `location`, and `epiweeks` are all required") + # Set up request + params = { + "endpoint": "norostat", + "auth": auth, + "location": location, + "epiweeks": Epidata._list(epiweeks), + } + # Make the API call + return Epidata._request(params) + + # Fetch NoroSTAT metadata + @staticmethod + def meta_norostat(auth): + """Fetch NoroSTAT metadata.""" + # Check parameters + if auth is None: + raise EpidataBadRequestException("`auth` is required") + # Set up request + params = { + "endpoint": "meta_norostat", + "auth": auth, + } + # Make the API call + return Epidata._request(params) + + # Fetch NIDSS flu data + @staticmethod + def nidss_flu(regions, epiweeks, issues=None, lag=None): + """Fetch NIDSS flu data.""" + # Check parameters + if regions is None or epiweeks is None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + if issues is not None and lag is not None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "nidss_flu", + "regions": Epidata._list(regions), + "epiweeks": Epidata._list(epiweeks), + } + if issues is not None: + params["issues"] = Epidata._list(issues) + if lag is not None: + params["lag"] = lag + # Make the API call + return Epidata._request(params) + + # Fetch NIDSS dengue data + @staticmethod + def nidss_dengue(locations, epiweeks): + """Fetch NIDSS dengue data.""" + # Check parameters + if locations is None or epiweeks is None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "nidss_dengue", + "locations": Epidata._list(locations), + "epiweeks": Epidata._list(epiweeks), + } + # Make the API call + return Epidata._request(params) + + # Fetch Delphi's forecast + @staticmethod + def delphi(system, epiweek): + """Fetch Delphi's forecast.""" + # Check parameters + if system is None or epiweek is None: + raise EpidataBadRequestException("`system` and `epiweek` are both required") + # Set up request + params = { + "endpoint": "delphi", + "system": system, + "epiweek": epiweek, + } + # Make the API call + return Epidata._request(params) + + # Fetch Delphi's digital surveillance sensors + @staticmethod + def sensors(auth, names, locations, epiweeks): + """Fetch Delphi's digital surveillance sensors.""" + # Check parameters + if names is None or locations is None or epiweeks is None: + raise EpidataBadRequestException( + "`names`, `locations`, and `epiweeks` are all required" + ) + # Set up request + params = { + "endpoint": "sensors", + "names": Epidata._list(names), + "locations": Epidata._list(locations), + "epiweeks": Epidata._list(epiweeks), + } + if auth is not None: + params["auth"] = auth + # Make the API call + return Epidata._request(params) + + # Fetch Delphi's dengue digital surveillance sensors + @staticmethod + def dengue_sensors(auth, names, locations, epiweeks): + """Fetch Delphi's digital surveillance sensors.""" + # Check parameters + if auth is None or names is None or locations is None or epiweeks is None: + raise EpidataBadRequestException( + "`auth`, `names`, `locations`, and `epiweeks` are all required" + ) + # Set up request + params = { + "endpoint": "dengue_sensors", + "auth": auth, + "names": Epidata._list(names), + "locations": Epidata._list(locations), + "epiweeks": Epidata._list(epiweeks), + } + # Make the API call + return Epidata._request(params) + + # Fetch Delphi's wILI nowcast + @staticmethod + def nowcast(locations, epiweeks): + """Fetch Delphi's wILI nowcast.""" + # Check parameters + if locations is None or epiweeks is None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "nowcast", + "locations": Epidata._list(locations), + "epiweeks": Epidata._list(epiweeks), + } + # Make the API call + return Epidata._request(params) + + # Fetch Delphi's dengue nowcast + @staticmethod + def dengue_nowcast(locations, epiweeks): + """Fetch Delphi's dengue nowcast.""" + # Check parameters + if locations is None or epiweeks is None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "dengue_nowcast", + "locations": Epidata._list(locations), + "epiweeks": Epidata._list(epiweeks), + } + # Make the API call + return Epidata._request(params) + + # Fetch API metadata + @staticmethod + def meta(): + """Fetch API metadata.""" + return Epidata._request({"endpoint": "meta"}) + + # Fetch Delphi's COVID-19 Surveillance Streams + @staticmethod + def covidcast( + data_source, + signals, + time_type, + geo_type, + time_values, + geo_value, + as_of=None, + issues=None, + lag=None, + **kwargs, + ): + """Fetch Delphi's COVID-19 Surveillance Streams""" + # also support old parameter name + if signals is None and "signal" in kwargs: + signals = kwargs["signal"] + # Check parameters + if None in (data_source, signals, time_type, geo_type, time_values, geo_value): + raise EpidataBadRequestException( + "`data_source`, `signals`, `time_type`, `geo_type`, " + "`time_values`, and `geo_value` are all required" + ) + if issues is not None and lag is not None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "endpoint": "covidcast", + "data_source": data_source, + "signals": Epidata._list(signals), + "time_type": time_type, + "geo_type": geo_type, + "time_values": Epidata._list(time_values), + } + + if isinstance(geo_value, (list, tuple)): + params["geo_values"] = ",".join(geo_value) + else: + params["geo_value"] = geo_value + if as_of is not None: + params["as_of"] = as_of + if issues is not None: + params["issues"] = Epidata._list(issues) + if lag is not None: + params["lag"] = lag + + if "format" in kwargs: + params["format"] = kwargs["format"] + + if "fields" in kwargs: + params["fields"] = kwargs["fields"] + + # Make the API call + return Epidata._request(params) + + # Fetch Delphi's COVID-19 Surveillance Streams metadata + @staticmethod + def covidcast_meta(): + """Fetch Delphi's COVID-19 Surveillance Streams metadata""" + return Epidata._request({"endpoint": "covidcast_meta"}) + + # Fetch COVID hospitalization data + @staticmethod + def covid_hosp(states, dates, issues=None, as_of=None): + """Fetch COVID hospitalization data.""" + # Check parameters + if states is None or dates is None: + raise EpidataBadRequestException("`states` and `dates` are both required") + # Set up request + params = { + "endpoint": "covid_hosp", + "states": Epidata._list(states), + "dates": Epidata._list(dates), + } + if issues is not None: + params["issues"] = Epidata._list(issues) + if as_of is not None: + params["as_of"] = as_of + # Make the API call + return Epidata._request(params) + + # Fetch COVID hospitalization data for specific facilities + @staticmethod + def covid_hosp_facility(hospital_pks, collection_weeks, publication_dates=None): + """Fetch COVID hospitalization data for specific facilities.""" + # Check parameters + if hospital_pks is None or collection_weeks is None: + raise EpidataBadRequestException( + "`hospital_pks` and `collection_weeks` are both required" + ) + # Set up request + params = { + "source": "covid_hosp_facility", + "hospital_pks": Epidata._list(hospital_pks), + "collection_weeks": Epidata._list(collection_weeks), + } + if publication_dates is not None: + params["publication_dates"] = Epidata._list(publication_dates) + # Make the API call + return Epidata._request(params) + + # Lookup COVID hospitalization facility identifiers + @staticmethod + def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_code=None): + """Lookup COVID hospitalization facility identifiers.""" + # Set up request + params = {"source": "covid_hosp_facility_lookup"} + if state is not None: + params["state"] = state + elif ccn is not None: + params["ccn"] = ccn + elif city is not None: + params["city"] = city + elif zip is not None: + params["zip"] = zip + elif fips_code is not None: + params["fips_code"] = fips_code + else: + raise EpidataBadRequestException( + "one of `state`, `ccn`, `city`, `zip`, or `fips_code` is required" + ) + # Make the API call + return Epidata._request(params) + + # Fetch Delphi's COVID-19 Nowcast sensors + @staticmethod + def covidcast_nowcast( + data_source, + signals, + sensor_names, + time_type, + geo_type, + time_values, + geo_value, + as_of=None, + issues=None, + lag=None, + **kwargs, + ): + """Fetch Delphi's COVID-19 Nowcast sensors""" + # Check parameters + # fmt: off + if None in (data_source, signals, time_type, geo_type, time_values, geo_value, sensor_names): + # fmt: on + raise EpidataBadRequestException( + "`data_source`, `signals`, `sensor_names`, `time_type`, `geo_type`, " + "`time_values`, and `geo_value` are all required" + ) + if issues is not None and lag is not None: + raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) + # Set up request + params = { + "source": "covidcast_nowcast", + "data_source": data_source, + "signals": Epidata._list(signals), + "sensor_names": Epidata._list(sensor_names), + "time_type": time_type, + "geo_type": geo_type, + "time_values": Epidata._list(time_values), + } + + if isinstance(geo_value, (list, tuple)): + params["geo_values"] = ",".join(geo_value) + else: + params["geo_value"] = geo_value + if as_of is not None: + params["as_of"] = as_of + if issues is not None: + params["issues"] = Epidata._list(issues) + if lag is not None: + params["lag"] = lag + + if "format" in kwargs: + params["format"] = kwargs["format"] + + # Make the API call + return Epidata._request(params) + + @staticmethod + def async_epidata(param_list, batch_size=50): + """Make asynchronous Epidata calls for a list of parameters.""" + + async def async_get(params, session): + """Helper function to make Epidata GET requests.""" + async with session.get(Epidata.BASE_URL, params=params) as response: + response.raise_for_status() + return await response.json(), params + + async def async_make_calls(param_combos): + """Helper function to asynchronously make and aggregate Epidata GET requests.""" + tasks = [] + connector = TCPConnector(limit=batch_size) + if isinstance(Epidata.auth, tuple): + auth = BasicAuth(login=Epidata.auth[0], password=Epidata.auth[1], encoding="utf-8") + else: + auth = Epidata.auth + async with ClientSession(connector=connector, headers=_HEADERS, auth=auth) as session: + for param in param_combos: + task = asyncio.ensure_future(async_get(param, session)) + tasks.append(task) + responses = await asyncio.gather(*tasks) + return responses + + loop = asyncio.get_event_loop() + future = asyncio.ensure_future(async_make_calls(param_list)) + responses = loop.run_until_complete(future) return responses - - loop = asyncio.get_event_loop() - future = asyncio.ensure_future(async_make_calls(param_list)) - responses = loop.run_until_complete(future) - return responses diff --git a/src/client/packaging/npm/package.json b/src/client/packaging/npm/package.json index 9e4e88b34..dd1d22611 100644 --- a/src/client/packaging/npm/package.json +++ b/src/client/packaging/npm/package.json @@ -2,7 +2,7 @@ "name": "delphi_epidata", "description": "Delphi Epidata API Client", "authors": "Delphi Group", - "version": "4.1.5", + "version": "4.1.6", "license": "MIT", "homepage": "https://github.com/cmu-delphi/delphi-epidata", "bugs": { diff --git a/src/client/packaging/pypi/delphi_epidata/__init__.py b/src/client/packaging/pypi/delphi_epidata/__init__.py index d24894719..cac388bec 100644 --- a/src/client/packaging/pypi/delphi_epidata/__init__.py +++ b/src/client/packaging/pypi/delphi_epidata/__init__.py @@ -1,4 +1,4 @@ from .delphi_epidata import Epidata -name = 'delphi_epidata' -__version__ = '4.1.5' +name = "delphi_epidata" +__version__ = "4.1.6" diff --git a/src/client/packaging/pypi/setup.py b/src/client/packaging/pypi/setup.py index 6838bccca..627e14753 100644 --- a/src/client/packaging/pypi/setup.py +++ b/src/client/packaging/pypi/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="delphi_epidata", - version="4.1.5", + version="4.1.6", author="David Farrow", author_email="dfarrow0@gmail.com", description="A programmatic interface to Delphi's Epidata API.", diff --git a/src/server/_common.py b/src/server/_common.py index f7c28c7ef..8633d07fd 100644 --- a/src/server/_common.py +++ b/src/server/_common.py @@ -11,7 +11,7 @@ from ._config import SECRET, REVERSE_PROXY_DEPTH from ._db import engine from ._exceptions import DatabaseErrorException, EpiDataException -from ._security import current_user, _is_public_route, resolve_auth_token, show_no_api_key_warning, update_key_last_time_used, ERROR_MSG_INVALID_KEY +from ._security import current_user, _is_public_route, resolve_auth_token, update_key_last_time_used, ERROR_MSG_INVALID_KEY app = Flask("EpiData", static_url_path="") @@ -128,11 +128,10 @@ def before_request_execute(): user_id=(user and user.id) ) - if not show_no_api_key_warning(): - if not _is_public_route() and api_key and not user: - # if this is a privleged endpoint, and an api key was given but it does not look up to a user, raise exception: - get_structured_logger("server_api").info("bad api key used", api_key=api_key) - raise Unauthorized(ERROR_MSG_INVALID_KEY) + if not _is_public_route() and api_key and not user: + # if this is a privleged endpoint, and an api key was given but it does not look up to a user, raise exception: + get_structured_logger("server_api").info("bad api key used", api_key=api_key) + raise Unauthorized(ERROR_MSG_INVALID_KEY) if request.path.startswith("/lib"): # files served from 'lib' directory don't need the database, so we can exit this early... diff --git a/src/server/_config.py b/src/server/_config.py index 77093454a..5d905c968 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -7,7 +7,7 @@ load_dotenv() -VERSION = "4.1.5" +VERSION = "4.1.6" MAX_RESULTS = int(10e6) MAX_COMPATIBILITY_RESULTS = int(3650) @@ -85,8 +85,6 @@ } NATION_REGION = "nat" -API_KEY_REQUIRED_STARTING_AT = date.fromisoformat(os.environ.get("API_KEY_REQUIRED_STARTING_AT", "2023-06-21")) -TEMPORARY_API_KEY = os.environ.get("TEMPORARY_API_KEY", "TEMP-API-KEY-EXPIRES-2023-06-28") # password needed for the admin application if not set the admin routes won't be available ADMIN_PASSWORD = os.environ.get("API_KEY_ADMIN_PASSWORD", "abc") # secret for the google form to give to the admin/register endpoint diff --git a/src/server/_limiter.py b/src/server/_limiter.py index 4bf72e05b..c54a2141c 100644 --- a/src/server/_limiter.py +++ b/src/server/_limiter.py @@ -1,5 +1,5 @@ from delphi.epidata.server.endpoints.covidcast_utils.dashboard_signals import DashboardSignals -from flask import Response, request, make_response, jsonify +from flask import Response, request from flask_limiter import Limiter, HEADERS from redis import Redis from werkzeug.exceptions import Unauthorized, TooManyRequests @@ -7,8 +7,9 @@ from ._common import app, get_real_ip_addr from ._config import RATE_LIMIT, RATELIMIT_STORAGE_URL, REDIS_HOST, REDIS_PASSWORD from ._exceptions import ValidationFailedException -from ._params import extract_dates, extract_integers, extract_strings -from ._security import _is_public_route, current_user, require_api_key, show_no_api_key_warning, resolve_auth_token, ERROR_MSG_RATE_LIMIT, ERROR_MSG_MULTIPLES +from ._params import extract_dates, extract_integers, extract_strings, parse_source_signal_sets +from ._security import _is_public_route, current_user, resolve_auth_token, ERROR_MSG_RATE_LIMIT, ERROR_MSG_MULTIPLES + def deduct_on_success(response: Response) -> bool: @@ -52,8 +53,9 @@ def get_multiples_count(request): if "window" in request.args.keys(): multiple_selection_allowed -= 1 for k, v in request.args.items(): - if v == "*": + if "*" in v: multiple_selection_allowed -= 1 + continue try: vals = multiples.get(k)(k) if len(vals) >= 2: @@ -70,16 +72,23 @@ def get_multiples_count(request): def check_signals_allowlist(request): signals_allowlist = {":".join(ss_pair) for ss_pair in DashboardSignals().srcsig_list()} - request_signals = [] - if "signal" in request.args.keys(): - request_signals += extract_strings("signal") - if "signals" in request.args.keys(): - request_signals += extract_strings("signals") - if "data_source" in request.args: - request_signals = [f"{request.args['data_source']}:{request_signal}" for request_signal in request_signals] + request_signals = set() + try: + source_signal_sets = parse_source_signal_sets() + except ValidationFailedException: + return False + for source_signal in source_signal_sets: + # source_signal.signal is expected to be eiter list or bool: + # in case of bool, we have wildcard signal -> return False as there are no chances that + # all signals from given source will be whitelisted + # in case of list, we have list of signals + if isinstance(source_signal.signal, bool): + return False + for signal in source_signal.signal: + request_signals.add(f"{source_signal.source}:{signal}") if len(request_signals) == 0: return False - return all([signal in signals_allowlist for signal in request_signals]) + return request_signals.issubset(signals_allowlist) def _resolve_tracking_key() -> str: @@ -108,23 +117,8 @@ def ratelimit_handler(e): return TooManyRequests(ERROR_MSG_RATE_LIMIT) -def requests_left(): - r = Redis(host=REDIS_HOST, password=REDIS_PASSWORD) - allowed_count, period = RATE_LIMIT.split("/") - try: - remaining_count = int(allowed_count) - int( - r.get(f"LIMITER/{_resolve_tracking_key()}/EpidataLimiter/{allowed_count}/1/{period}") - ) - except TypeError: - return 1 - return remaining_count - - @limiter.request_filter def _no_rate_limit() -> bool: - if show_no_api_key_warning(): - # no rate limit in phase 0 - return True if _is_public_route(): # no rate limit for public routes return True @@ -132,15 +126,6 @@ def _no_rate_limit() -> bool: # no rate limit if user is registered return True - if not require_api_key(): - # we are in phase 1 or 2 - if requests_left() > 0: - # ...and user is below rate limit, we still want to record this query for the rate computation... - return False - # ...otherwise, they have exceeded the limit, but we still want to allow them through - return True - - # phase 3 (full api-keys behavior) multiples = get_multiples_count(request) if multiples < 0: # too many multiples diff --git a/src/server/_printer.py b/src/server/_printer.py index 336b650cc..5616787a2 100644 --- a/src/server/_printer.py +++ b/src/server/_printer.py @@ -2,15 +2,12 @@ from io import StringIO from typing import Any, Dict, Iterable, List, Optional, Union -from flask import Response, jsonify, stream_with_context, request +from flask import Response, jsonify, stream_with_context from flask.json import dumps import orjson from ._config import MAX_RESULTS, MAX_COMPATIBILITY_RESULTS -# TODO: remove warnings after once we are past the API_KEY_REQUIRED_STARTING_AT date -from ._security import show_hard_api_key_warning, show_soft_api_key_warning, ROLLOUT_WARNING_RATE_LIMIT, ROLLOUT_WARNING_MULTIPLES, _ROLLOUT_WARNING_AD_FRAGMENT, PHASE_1_2_STOPGAP from ._common import is_compatibility_mode, log_info_with_request -from ._limiter import requests_left, get_multiples_count from delphi.epidata.common.logger import get_structured_logger @@ -25,15 +22,7 @@ def print_non_standard(format: str, data): message = "no results" result = -2 else: - warning = "" - if show_hard_api_key_warning(): - if requests_left() == 0: - warning = f"{ROLLOUT_WARNING_RATE_LIMIT}" - if get_multiples_count(request) < 0: - warning = f"{warning} {ROLLOUT_WARNING_MULTIPLES}" - if requests_left() == 0 or get_multiples_count(request) < 0: - warning = f"{warning} {_ROLLOUT_WARNING_AD_FRAGMENT} {PHASE_1_2_STOPGAP}" - message = warning.strip() or "success" + message = "success" result = 1 if result == -1 and is_compatibility_mode(): return jsonify(dict(result=result, message=message)) @@ -127,40 +116,21 @@ class ClassicPrinter(APrinter): """ def _begin(self): - if is_compatibility_mode() and not show_hard_api_key_warning(): + if is_compatibility_mode(): return "{ " - r = '{ "epidata": [' - if show_hard_api_key_warning(): - warning = "" - if requests_left() == 0: - warning = f"{warning} {ROLLOUT_WARNING_RATE_LIMIT}" - if get_multiples_count(request) < 0: - warning = f"{warning} {ROLLOUT_WARNING_MULTIPLES}" - if requests_left() == 0 or get_multiples_count(request) < 0: - warning = f"{warning} {_ROLLOUT_WARNING_AD_FRAGMENT} {PHASE_1_2_STOPGAP}" - if warning != "": - return f'{r} "{warning.strip()}",' - return r + return '{ "epidata": [' def _format_row(self, first: bool, row: Dict): - if first and is_compatibility_mode() and not show_hard_api_key_warning(): + if first and is_compatibility_mode(): sep = b'"epidata": [' else: sep = b"," if not first else b"" return sep + orjson.dumps(row) def _end(self): - warning = "" - if show_soft_api_key_warning(): - if requests_left() == 0: - warning = f"{warning} {ROLLOUT_WARNING_RATE_LIMIT}" - if get_multiples_count(request) < 0: - warning = f"{warning} {ROLLOUT_WARNING_MULTIPLES}" - if requests_left() == 0 or get_multiples_count(request) < 0: - warning = f"{warning} {_ROLLOUT_WARNING_AD_FRAGMENT} {PHASE_1_2_STOPGAP}" - message = warning.strip() or "success" + message = "success" prefix = "], " - if self.count == 0 and is_compatibility_mode() and not show_hard_api_key_warning(): + if self.count == 0 and is_compatibility_mode(): # no array to end prefix = "" @@ -194,7 +164,7 @@ def _format_row(self, first: bool, row: Dict): self._tree[group].append(row) else: self._tree[group] = [row] - if first and is_compatibility_mode() and not show_hard_api_key_warning(): + if first and is_compatibility_mode(): return b'"epidata": [' return None @@ -205,10 +175,7 @@ def _end(self): tree = orjson.dumps(self._tree) self._tree = dict() r = super(ClassicTreePrinter, self)._end() - r = tree + r - if show_hard_api_key_warning() and (requests_left() == 0 or get_multiples_count(request) < 0): - r = b", " + r - return r + return tree + r class CSVPrinter(APrinter): @@ -243,17 +210,6 @@ def _format_row(self, first: bool, row: Dict): columns = list(row.keys()) self._writer = DictWriter(self._stream, columns, lineterminator="\n") self._writer.writeheader() - if show_hard_api_key_warning(): - warning = "" - if requests_left() == 0: - warning = f"{warning} {ROLLOUT_WARNING_RATE_LIMIT}" - if get_multiples_count(request) < 0: - warning = f"{warning} {ROLLOUT_WARNING_MULTIPLES}" - if requests_left() == 0 or get_multiples_count(request) < 0: - warning = f"{warning} {_ROLLOUT_WARNING_AD_FRAGMENT} {PHASE_1_2_STOPGAP}" - if warning.strip() != "": - self._writer.writerow({columns[0]: warning}) - self._writer.writerow(row) # remove the stream content to print just one line at a time @@ -274,18 +230,7 @@ class JSONPrinter(APrinter): """ def _begin(self): - r = b"[" - if show_hard_api_key_warning(): - warning = "" - if requests_left() == 0: - warning = f"{warning} {ROLLOUT_WARNING_RATE_LIMIT}" - if get_multiples_count(request) < 0: - warning = f"{warning} {ROLLOUT_WARNING_MULTIPLES}" - if requests_left() == 0 or get_multiples_count(request) < 0: - warning = f"{warning} {_ROLLOUT_WARNING_AD_FRAGMENT} {PHASE_1_2_STOPGAP}" - if warning.strip() != "": - r = b'["' + bytes(warning, "utf-8") + b'",' - return r + return b"[" def _format_row(self, first: bool, row: Dict): sep = b"," if not first else b"" @@ -303,19 +248,6 @@ class JSONLPrinter(APrinter): def make_response(self, gen, headers=None): return Response(gen, mimetype=" text/plain; charset=utf8", headers=headers) - def _begin(self): - if show_hard_api_key_warning(): - warning = "" - if requests_left() == 0: - warning = f"{warning} {ROLLOUT_WARNING_RATE_LIMIT}" - if get_multiples_count(request) < 0: - warning = f"{warning} {ROLLOUT_WARNING_MULTIPLES}" - if requests_left() == 0 or get_multiples_count(request) < 0: - warning = f"{warning} {_ROLLOUT_WARNING_AD_FRAGMENT} {PHASE_1_2_STOPGAP}" - if warning.strip() != "": - return bytes(warning, "utf-8") + b"\n" - return None - def _format_row(self, first: bool, row: Dict): # each line is a JSON file with a new line to separate them return orjson.dumps(row, option=orjson.OPT_APPEND_NEWLINE) @@ -338,4 +270,4 @@ def create_printer(format: str) -> APrinter: return CSVPrinter() if format == "jsonl": return JSONLPrinter() - return ClassicPrinter() + return ClassicPrinter() \ No newline at end of file diff --git a/src/server/_security.py b/src/server/_security.py index b40ac445e..38294eb10 100644 --- a/src/server/_security.py +++ b/src/server/_security.py @@ -9,27 +9,13 @@ from werkzeug.local import LocalProxy from ._config import ( - API_KEY_REQUIRED_STARTING_AT, REDIS_HOST, REDIS_PASSWORD, API_KEY_REGISTRATION_FORM_LINK_LOCAL, - TEMPORARY_API_KEY, URL_PREFIX, ) from .admin.models import User -API_KEY_HARD_WARNING = API_KEY_REQUIRED_STARTING_AT - timedelta(days=14) -API_KEY_SOFT_WARNING = API_KEY_HARD_WARNING - timedelta(days=14) - -# rollout warning messages -ROLLOUT_WARNING_RATE_LIMIT = "This request exceeded the rate limit on anonymous requests, which will be enforced starting {}.".format(API_KEY_REQUIRED_STARTING_AT) -ROLLOUT_WARNING_MULTIPLES = "This request exceeded the anonymous limit on selected multiples, which will be enforced starting {}.".format(API_KEY_REQUIRED_STARTING_AT) -_ROLLOUT_WARNING_AD_FRAGMENT = "To be exempt from this limit, authenticate your requests with a free API key, now available at {}.".format(API_KEY_REGISTRATION_FORM_LINK_LOCAL) - -PHASE_1_2_STOPGAP = ( - "A temporary public key `{}` is available for use between now and {} to give you time to register or adapt your requests without this message continuing to break your systems." -).format(TEMPORARY_API_KEY, (API_KEY_REQUIRED_STARTING_AT + timedelta(days=7))) - # steady-state error messages ERROR_MSG_RATE_LIMIT = "Rate limit exceeded for anonymous queries. To remove this limit, register a free API key at {}".format(API_KEY_REGISTRATION_FORM_LINK_LOCAL) @@ -54,30 +40,6 @@ def resolve_auth_token() -> Optional[str]: return None -def show_no_api_key_warning() -> bool: - # aka "phase 0" - n = date.today() - return not current_user and n < API_KEY_SOFT_WARNING - - -def show_soft_api_key_warning() -> bool: - # aka "phase 1" - n = date.today() - return not current_user and API_KEY_SOFT_WARNING <= n < API_KEY_HARD_WARNING - - -def show_hard_api_key_warning() -> bool: - # aka "phase 2" - n = date.today() - return not current_user and API_KEY_HARD_WARNING <= n < API_KEY_REQUIRED_STARTING_AT - - -def require_api_key() -> bool: - # aka "phase 3" - n = date.today() - return API_KEY_REQUIRED_STARTING_AT <= n - - def _get_current_user(): if "user" not in g: api_key = resolve_auth_token() diff --git a/src/server/utils/dates.py b/src/server/utils/dates.py index e810e2146..4d6c242c9 100644 --- a/src/server/utils/dates.py +++ b/src/server/utils/dates.py @@ -91,20 +91,19 @@ def time_values_to_ranges(values: Optional[TimeValues]) -> Optional[TimeValues]: """ logger = get_structured_logger('server_utils') if not values or len(values) <= 1: - logger.info("List of dates looks like 0-1 elements, nothing to optimize", time_values=values) + logger.debug("List of dates looks like 0-1 elements, nothing to optimize", time_values=values) return values # determine whether the list is of days (YYYYMMDD) or weeks (YYYYWW) based on first element first_element = values[0][0] if isinstance(values[0], tuple) else values[0] if guess_time_value_is_day(first_element): - # TODO: reduce this and other date logging to DEBUG after prod metrics gathered - logger.info("Treating time value as day", time_value=first_element) + logger.debug("Treating time value as day", time_value=first_element) return days_to_ranges(values) elif guess_time_value_is_week(first_element): - logger.info("Treating time value as week", time_value=first_element) + logger.debug("Treating time value as week", time_value=first_element) return weeks_to_ranges(values) else: - logger.info("Time value unclear, not optimizing", time_value=first_element) + logger.debug("Time value unclear, not optimizing", time_value=first_element) return values def days_to_ranges(values: TimeValues) -> TimeValues: @@ -147,9 +146,9 @@ def _to_ranges(values: TimeValues, value_to_date: Callable, date_to_value: Calla else: ranges.append((date_to_value(m[0]), date_to_value(m[1]))) - get_structured_logger('server_utils').info("Optimized list of date values", original=values, optimized=ranges, original_length=len(values), optimized_length=len(ranges)) + get_structured_logger('server_utils').debug("Optimized list of date values", original=values, optimized=ranges, original_length=len(values), optimized_length=len(ranges)) return ranges except Exception as e: - get_structured_logger('server_utils').error('bad input to date ranges', time_values=values, exception=e) + get_structured_logger('server_utils').debug('bad input to date ranges', time_values=values, exception=e) return values