Flask based API

.dockerignore

@@ -0,0 +1,6 @@
+/delphi-epidata
+/.mypy_cache
+/.github
+/docs
+__pycache__
+/node_modules

.env.example

@@ -0,0 +1,18 @@
+FLASK_DEBUG=True
+SQLALCHEMY_DATABASE_URI=sqlite:///test.db
+FLASK_SECRET=abc
+SECRET_TWITTER=abc
+SECRET_GHT=abc
+SECRET_FLUVIEW=abc
+SECRET_CDC=abc
+SECRET_SENSORS=abc
+SECRET_SENSOR_TWTR=abc
+SECRET_SENSOR_GFT=abc
+SECRET_SENSOR_GHT=abc
+SECRET_SENSOR_GHTJ=abc
+SECRET_SENSOR_CDC=abc
+SECRET_SENSOR_QUID=abc
+SECRET_SENSOR_WIKI=abc
+SECRET_QUIDEL=abc
+SECRET_NOROSTAT=abc
+SECRET_AFHSB=abc

.github/workflows/ci.yaml

@@ -51,17 +51,19 @@ jobs:
 
       - name: Build docker images
         run: |
-          docker build -t delphi_web -f repos/delphi/operations/dev/docker/web/Dockerfile .
           docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile .
           docker build -t delphi_python -f repos/delphi/operations/dev/docker/python/Dockerfile .
-          docker build -t delphi_web_epidata -f ./repos/delphi/delphi-epidata/dev/docker/web/epidata/Dockerfile .
           docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
+          docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
+          cd ./repos/delphi/delphi-epidata
+          docker build -t delphi_web_epidata -f ./devops/Dockerfile .
+          cd ../../../
 
       - name: Start services
         run: |
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata delphi_database_epidata
-          docker run --rm -d -p 10080:80 --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker run --rm -d -p 10080:80 --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
           docker ps
 
       - run: |
@@ -71,13 +73,67 @@ jobs:
           sleep 10s
       - name: Run Unit Tests
         run: |
-          docker run --rm --network delphi-net delphi_python python3 -m undefx.py3tester.py3tester --use-exit-code repos/delphi/delphi-epidata/tests
+          docker run --rm --network delphi-net --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests
 
       - name: Run Integration Tests
         run: |
-          docker run --rm --network delphi-net delphi_python python3 -m undefx.py3tester.py3tester --use-exit-code repos/delphi/delphi-epidata/integrations
+          docker run --rm --network delphi-net delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/integrations
 
       - name: Clean Up
         run: |
           docker stop delphi_database_epidata delphi_web_epidata
-          docker network remove delphi-net
+          docker network remove delphi-net
+
+  image:
+    # only on main and dev branch
+    if: startsWith(github.ref, 'refs/heads/main') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/sgratzl/flaskx'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: cmu-delphi-deploy-machine
+          password: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+      - name: Build Image
+        env:
+          DEVOPS_DOCKER_FILE: ./devops/Dockerfile
+        run: |
+          docker build -t repo --file ${DEVOPS_DOCKER_FILE} .
+      - name: Resolve Tag
+        id: tagname
+        run: |
+          baseRef="${GITHUB_REF#*/}"
+          imageTag="${baseRef#*/}"
+          case "${baseRef}" in
+          main)
+            imageTag="latest"
+            ;;
+          *)
+            imageTag="${baseRef//\//_}" # replace `/` with `_` in branch name
+            ;;
+          esac
+          echo "::set-output name=tag::$imageTag"
+          echo "::set-output name=repo::ghcr.io/${{ github.repository }}"
+      - name: Push Dev Tag
+        run: |
+          docker tag repo ${{ steps.tagname.outputs.repo }}:${{ steps.tagname.outputs.tag }}
+          docker push ${{ steps.tagname.outputs.repo }}:${{ steps.tagname.outputs.tag }}
+      # - name: Extract version
+      #   if: startsWith(github.ref, 'refs/heads/main')
+      #   id: extract_version
+      #   run: node -pe "'::set-output name=version::' + require('./package.json').version"
+      - name: Trigger Webhook
+        run: |
+          # trigger a webhook update
+          curl -H "Authorization: Bearer ${{ secrets.DELPHI_DEPLOY_WEBHOOK_TOKEN }}" \
+               -X POST ${{ secrets.DELPHI_DEPLOY_WEBHOOK_URL }} \
+               -H "Content-Type: application/x-www-form-urlencoded" \
+               -d "repository=${{ steps.tagname.outputs.repo }}&tag=${{ steps.tagname.outputs.tag }}"
+      # - name: Push Version Tag Tag
+      #   if: startsWith(github.ref, 'refs/heads/main')
+      #   run: |
+      #     docker tag repo ${{ steps.tagname.outputs.repo }}:v${{ steps.extract_version.outputs.version }}
+      #     docker push ${{ steps.tagname.outputs.repo }}:v${{ steps.extract_version.outputs.version }}

.gitignore

@@ -1,3 +1,8 @@
 __pycache__/
 .DS_Store
-
+/.vscode
+/delphi-epidata
+/.env
+*.db
+/build
+/node_modules

build.js

@@ -0,0 +1,6 @@
+const { process, minimizeJs, rename } = require('delphi-cmu-buildtools');
+
+Promise.all([
+    process('*.+(coffee|js|py|R)', [], { srcDir: './src/client', dstDir: './build/lib' }),
+    process('*.js', [minimizeJs(), rename((f) => f.replace('.js', '.min.js'))], { srcDir: './src/client', dstDir: './build/lib' }),
+]).then((r) => console.log(r.flat()));

deploy.json

@@ -4,7 +4,8 @@
   "paths": {
     "root_web": "/var/www/html/epidata",
     "auto_web": "/home/automation/public_html/public",
-    "package": "../delphi/epidata"
+    "package": "../delphi/epidata",
+    "stasis": "../stasis"
   },
   "actions": [
 
@@ -43,14 +44,28 @@
       "add-header-comment": true
     },
 
-    "// server - API tests",
+    "// server",
     {
       "type": "move",
       "src": "src/server/",
       "dst": "[[package]]/server/",
       "match": "^.*\\.(py)$",
       "add-header-comment": true
     },
+    {
+      "type": "move",
+      "src": "src/server/endpoints",
+      "dst": "[[package]]/server/endpoints/",
+      "match": "^.*\\.(py)$",
+      "add-header-comment": true
+    },
+    {
+      "type": "move",
+      "src": "src/server/endpoints/covidcast_utils",
+      "dst": "[[package]]/server/endpoints/covidcast_utils/",
+      "match": "^.*\\.(py)$",
+      "add-header-comment": true
+    },
 
     "// acquisition - fluview",
     {
@@ -246,6 +261,22 @@
       "add-header-comment": true
     },
 
+    "// move flask tests out of the way",
+    {
+      "type": "move",
+      "src": "tests/server/",
+      "dst": "[[stasis]]/tests/",
+      "match": "^.*\\.(py)$",
+      "recursive": true
+    },
+    {
+      "type": "move",
+      "src": "integrations/server/",
+      "dst": "[[stasis]]/integrations/",
+      "match": "^.*\\.(py)$",
+      "recursive": true
+    },
+
     "// run unit and coverage tests",
     {"type": "py3test"}
 

dev/docker/python/Dockerfile

@@ -0,0 +1,4 @@
+# start with the `delphi_python` image
+FROM delphi_python
+
+RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.txt -r repos/delphi/delphi-epidata/requirements.dev.txt

devops/Dockerfile

@@ -0,0 +1,26 @@
+FROM node:lts-buster AS builder
+WORKDIR /src
+COPY . /src
+RUN npm ci && npm run build
+
+FROM tiangolo/meinheld-gunicorn:python3.7
+LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epidata
+# use delphi's timezome
+RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
+
+COPY requirements.txt /app
+RUN pip install --no-cache-dir -r requirements.txt
+
+# disable python stdout buffering
+ENV PYTHONUNBUFFERED 1
+
+COPY ./devops/gunicorn_conf.py /app
+COPY ./devops/start_wrapper.sh /
+COPY ./src/server/ /app/app/
+COPY --from=builder ./src/build/lib/ /app/app/lib/
+RUN rm -rf /app/app/__pycache__ /app/app/*.php \
+      && chmod -R o+r /app/app \
+      && chmod 755 /start_wrapper.sh
+
+ENTRYPOINT [ "/entrypoint.sh" ]
+CMD [ "/start_wrapper.sh" ]

devops/gunicorn_conf.py

@@ -0,0 +1,53 @@
+from __future__ import print_function
+
+import json
+import multiprocessing
+import os
+
+workers_per_core_str = os.getenv("WORKERS_PER_CORE", "2")
+web_concurrency_str = os.getenv("WEB_CONCURRENCY", None)
+host = os.getenv("HOST", "0.0.0.0")
+port = os.getenv("PORT", "80")
+bind_env = os.getenv("BIND", None)
+use_loglevel = os.getenv("LOG_LEVEL", "info")
+if bind_env:
+    use_bind = bind_env
+else:
+    use_bind = "{host}:{port}".format(host=host, port=port)
+
+cores = multiprocessing.cpu_count()
+workers_per_core = float(workers_per_core_str)
+default_web_concurrency = workers_per_core * cores
+if web_concurrency_str:
+    web_concurrency = int(web_concurrency_str)
+    assert web_concurrency > 0
+else:
+    web_concurrency = int(default_web_concurrency)
+
+# Gunicorn config variables
+loglevel = use_loglevel
+workers = web_concurrency
+bind = use_bind
+keepalive = 120
+
+errorlog = "-"
+accesslog = "-"
+capture_output = True
+enable_stdio_inheritance = True
+
+# disable limit request line till 414 issue is resolved
+limit_request_line = 0
+limit_request_field_size = 0
+
+
+# For debugging and testing
+log_data = {
+    "loglevel": loglevel,
+    "workers": workers,
+    "bind": bind,
+    # Additional, non-gunicorn variables
+    "workers_per_core": workers_per_core,
+    "host": host,
+    "port": port,
+}
+print(json.dumps(log_data), flush=True)

devops/start_wrapper.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+set -e
+
+# If a New Relic license key is found then we start with custom New Relic
+# commands, otherwise we start via start.sh.
+if [ -z "${NEW_RELIC_LICENSE_KEY}" ]; then
+  sh /start.sh
+else
+  newrelic-admin run-program /start.sh
+fi

integrations/acquisition/covid_hosp/facility/test_scenarios.py

@@ -6,7 +6,7 @@
 
 # first party
 from delphi.epidata.acquisition.covid_hosp.common.database import Database
-from delphi.epidata.acquisition.covid_hosp.common.test_utils import TestUtils
+from delphi.epidata.acquisition.covid_hosp.common.test_utils import UnitTestUtils
 from delphi.epidata.client.delphi_epidata import Epidata
 from delphi.epidata.acquisition.covid_hosp.facility.update import Update
 import delphi.operations.secrets as secrets
@@ -24,7 +24,7 @@ def setUp(self):
     """Perform per-test setup."""
 
     # configure test data
-    self.test_utils = TestUtils(__file__)
+    self.test_utils = UnitTestUtils(__file__)
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'

integrations/acquisition/covid_hosp/state_daily/test_scenarios.py

@@ -10,7 +10,7 @@
 
 # first party
 from delphi.epidata.acquisition.covid_hosp.common.database import Database
-from delphi.epidata.acquisition.covid_hosp.common.test_utils import TestUtils
+from delphi.epidata.acquisition.covid_hosp.common.test_utils import UnitTestUtils
 from delphi.epidata.client.delphi_epidata import Epidata
 from delphi.epidata.acquisition.covid_hosp.state_daily.update import Update
 from delphi.epidata.acquisition.covid_hosp.state_daily.network import Network
@@ -27,7 +27,7 @@ def setUp(self):
     """Perform per-test setup."""
 
     # configure test data
-    self.test_utils = TestUtils(__file__)
+    self.test_utils = UnitTestUtils(__file__)
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'

integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py

@@ -6,7 +6,7 @@
 
 # first party
 from delphi.epidata.acquisition.covid_hosp.common.database import Database
-from delphi.epidata.acquisition.covid_hosp.common.test_utils import TestUtils
+from delphi.epidata.acquisition.covid_hosp.common.test_utils import UnitTestUtils
 from delphi.epidata.client.delphi_epidata import Epidata
 from delphi.epidata.acquisition.covid_hosp.state_timeseries.update import Update
 import delphi.operations.secrets as secrets
@@ -25,7 +25,7 @@ def setUp(self):
     """Perform per-test setup."""
 
     # configure test data
-    self.test_utils = TestUtils(__file__)
+    self.test_utils = UnitTestUtils(__file__)
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'