Improved API keys logic

.dockerignore

@@ -1,6 +1,7 @@
 /delphi-epidata
-/.mypy_cache
+**/.mypy_cache
 /.github
 /docs
-__pycache__
-/node_modules
+**/__pycache__
+**/.pytest_cache
+**/node_modules

.env.example

@@ -1,18 +1,8 @@
 FLASK_DEBUG=True
 SQLALCHEMY_DATABASE_URI=sqlite:///test.db
 FLASK_SECRET=abc
-SECRET_TWITTER=abc
-SECRET_GHT=abc
-SECRET_FLUVIEW=abc
-SECRET_CDC=abc
-SECRET_SENSORS=abc
-SECRET_SENSOR_TWTR=abc
-SECRET_SENSOR_GFT=abc
-SECRET_SENSOR_GHT=abc
-SECRET_SENSOR_GHTJ=abc
-SECRET_SENSOR_CDC=abc
-SECRET_SENSOR_QUID=abc
-SECRET_SENSOR_WIKI=abc
-SECRET_QUIDEL=abc
-SECRET_NOROSTAT=abc
-SECRET_AFHSB=abc
+#API_REQUIRED_STARTING_AT=2021-07-30
+API_KEY_ADMIN_PASSWORD=abc
+API_KEY_REGISTER_WEBHOOK_TOKEN=abc
+RECAPTCHA_SITE_KEY
+RECAPTCHA_SECRET_KEY

.github/workflows/ci.yaml

@@ -53,6 +53,7 @@ jobs:
         run: |
           docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
           docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
+          sudo docker build -t delphi_redis_instance -f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
           cd ./repos/delphi/delphi-epidata
           docker build -t delphi_web_epidata -f ./devops/Dockerfile .
           cd ../../../
@@ -63,7 +64,8 @@ jobs:
         run: |
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
-          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "RATELIMIT_STORAGE_URL=redis://delphi_redis_instance:6379" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker run --rm -p 6379:6379 --network delphi-net --name delphi_redis_instance delphi_redis_instance
           docker ps
 
       - run: |

dev/docker/python/Dockerfile

@@ -5,6 +5,7 @@ WORKDIR /usr/src/app
 COPY repos repos
 COPY repos/delphi/delphi-epidata/dev/docker/python/setup.sh .
 
+
 RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime && \
     chmod -R o+r repos/ && \
     bash setup.sh && \

dev/docker/redis/Dockerfile

@@ -0,0 +1,3 @@
+FROM redis
+
+CMD ["redis-server"]

devops/Dockerfile

@@ -23,7 +23,6 @@ RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime \
 # the file /tmp/requirements.txt is created in the parent docker definition.  (see:
 #   https://github.com/tiangolo/meinheld-gunicorn-docker/blob/master/docker-images/python3.8.dockerfile#L5 )
 # this combined requirements installation ensures all version constrants are accounted for.
-
 # disable python stdout buffering
 ENV PYTHONUNBUFFERED 1
 

integrations/acquisition/covid_hosp/facility/test_scenarios.py

@@ -29,6 +29,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -40,6 +41,8 @@ def setUp(self):
         cur.execute('truncate table covid_hosp_facility')
         cur.execute('truncate table covid_hosp_facility_key')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, tracking, registered) values ("key", 1, 1)')
 
   @freeze_time("2021-03-16")
   def test_acquire_dataset(self):

integrations/acquisition/covid_hosp/state_daily/test_scenarios.py

@@ -33,6 +33,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -43,6 +44,8 @@ def setUp(self):
       with db.new_cursor() as cur:
         cur.execute('truncate table covid_hosp_state_timeseries')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, tracking, registered) values("key", 1, 1)')
 
   @freeze_time("2021-03-16")
   def test_acquire_dataset(self):

integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py

@@ -29,6 +29,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -39,6 +40,8 @@ def setUp(self):
       with db.new_cursor() as cur:
         cur.execute('truncate table covid_hosp_state_timeseries')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, tracking, registered) values ("key", 1, 1)')
 
   @freeze_time("2021-03-17")
   def test_acquire_dataset(self):

integrations/acquisition/covidcast/test_covidcast_meta_caching.py

@@ -60,12 +60,20 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = BASE_URL
+    Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):
     """Perform per-test teardown."""
     self.cur.close()
     self.cnx.close()
 
+  @staticmethod
+  def _make_request():
+    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
+    response = requests.get(Epidata.BASE_URL, params=params, auth=Epidata.auth)
+    response.raise_for_status()
+    return response.json()
+
   def test_caching(self):
     """Populate, query, cache, query, and verify the cache."""
 
@@ -147,10 +155,7 @@ def test_caching(self):
     self.cnx.commit()
 
     # fetch the cached version (manually)
-    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
-    response = requests.get(BASE_URL, params=params)
-    response.raise_for_status()
-    epidata4 = response.json()
+    epidata4 = self._make_request()
 
     # make sure the cache was actually served
     self.assertEqual(epidata4, {
@@ -170,10 +175,7 @@ def test_caching(self):
     self.cnx.commit()
 
     # fetch the cached version (manually)
-    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
-    response = requests.get(BASE_URL, params=params)
-    response.raise_for_status()
-    epidata5 = response.json()
+    epidata5 = self._make_request()
 
     # make sure the cache was returned anyhow
     self.assertEqual(epidata4, epidata5)

integrations/acquisition/covidcast/test_csv_uploading.py

@@ -57,6 +57,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):
     """Perform per-test teardown."""

integrations/acquisition/covidcast_nowcast/test_csv_uploading.py

@@ -41,6 +41,8 @@ def setUp(self):
         database='epidata')
     cur = cnx.cursor()
     cur.execute('truncate table covidcast_nowcast')
+    cur.execute('delete from api_user')
+    cur.execute('insert into api_user(api_key, tracking, registered) values ("key", 1, 1)')
     cnx.commit()
     cur.close()
 
@@ -54,6 +56,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):
     """Perform per-test teardown."""

integrations/client/test_delphi_epidata.py

@@ -41,6 +41,7 @@ def localSetUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'

integrations/client/test_nowcast.py

@@ -28,6 +28,8 @@ def setUp(self):
     cur = cnx.cursor()
 
     cur.execute('truncate table covidcast_nowcast')
+    cur.execute('delete from api_user')
+    cur.execute('insert into api_user(api_key, tracking, registered) values ("key", 1, 1)')
 
     cnx.commit()
     cur.close()
@@ -38,6 +40,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'

integrations/server/test_covid_hosp.py

@@ -17,6 +17,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -27,6 +28,8 @@ def setUp(self):
       with db.new_cursor() as cur:
         cur.execute('truncate table covid_hosp_state_timeseries')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, tracking, registered) values ("key", 1, 1)')
 
 
   def insert_issue(self, cur, issue, value, record_type):

integrations/server/test_covidcast.py

@@ -6,15 +6,12 @@
 
 # third party
 import mysql.connector
-import requests
 
 # first party
 from delphi_utils import Nans
 from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow
 from delphi.epidata.client.delphi_epidata import Epidata
 
-# use the local instance of the Epidata API
-BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
 
 class CovidcastTests(CovidcastBase):
   """Tests the `covidcast` endpoint."""
@@ -25,7 +22,9 @@ def localSetUp(self):
 
   def request_based_on_row(self, row: CovidcastTestRow, **kwargs):
     params = self.params_from_row(row, endpoint='covidcast', **kwargs)
-    Epidata.BASE_URL = BASE_URL
+    # use the local instance of the Epidata API
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
     response = Epidata.covidcast(**params) 
 
     return response

integrations/server/test_covidcast_endpoints.py

@@ -15,6 +15,7 @@
 # use the local instance of the Epidata API
 BASE_URL = "http://delphi_web_epidata/epidata/covidcast"
 BASE_URL_OLD = "http://delphi_web_epidata/epidata/api.php"
+AUTH = ('epidata', 'key')
 
 
 class CovidcastEndpointTests(CovidcastBase):
@@ -36,7 +37,7 @@ def _fetch(self, endpoint="/", is_compatibility=False, **params):
                 params.setdefault("data_source", params.get("source"))
         else:
             url = f"{BASE_URL}{endpoint}"
-        response = requests.get(url, params=params)
+        response = requests.get(url, params=params, auth=AUTH)
         response.raise_for_status()
         return response.json()