Merge pull request #954 from cmu-delphi/v4-schema-revisions-release-prep

Epidata v4.1

Author: melange396

.github/workflows/ci.yaml

@@ -62,7 +62,7 @@ jobs:
       - name: Start services
         run: |
           docker network create --driver bridge delphi-net
-          docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata delphi_database_epidata
+          docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
           docker run --rm -d -p 10080:80 --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
           docker ps
 
@@ -108,7 +108,9 @@ jobs:
   image:
     needs: build
     # only on main and dev branch
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
+    #if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' # TODO: Revert after v4 migration is complete
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/v4-schema-revisions-release-prep'
+
     runs-on: ubuntu-latest
     steps:
       - name: Check out code

.gitignore

@@ -1,4 +1,7 @@
 __pycache__/
+*.pyc
+*~
+\#*#
 .DS_Store
 /.vscode
 /delphi-epidata

dev/docker/database/epidata/Dockerfile

@@ -1,5 +1,14 @@
-# start with the `delphi_database` image
-FROM delphi_database
+# start with a standard percona mysql image
+FROM percona:ps-8
+
+# percona exits with the mysql user but we need root for additional setup
+USER root
+
+# use delphi's timezome
+RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
+
+# specify a development-only password for the database user "root"
+ENV MYSQL_ROOT_PASSWORD pass
 
 # create the `epidata` database
 ENV MYSQL_DATABASE epidata
@@ -8,8 +17,17 @@ ENV MYSQL_DATABASE epidata
 ENV MYSQL_USER user
 ENV MYSQL_PASSWORD pass
 
+# provide DDL which will configure dev environment at container startup
+COPY repos/delphi/delphi-epidata/dev/docker/database/epidata/_init.sql /docker-entrypoint-initdb.d/
+
 # provide DDL which will create empty tables at container startup
 COPY repos/delphi/delphi-epidata/src/ddl/*.sql /docker-entrypoint-initdb.d/
 
+# provide additional configuration needed for percona
+COPY repos/delphi/delphi-epidata/dev/docker/database/mysql.d/*.cnf /etc/my.cnf.d/
+
 # grant access to SQL scripts
 RUN chmod o+r /docker-entrypoint-initdb.d/*.sql
+
+# restore mysql user for percona
+USER mysql

dev/docker/database/epidata/_init.sql

@@ -0,0 +1,2 @@
+CREATE DATABASE covid;
+GRANT ALL ON covid.* TO 'user';

dev/docker/database/mysql.d/my.cnf

@@ -0,0 +1,2 @@
+[mysqld]
+default_authentication_plugin=mysql_native_password

dev/local/Makefile

@@ -10,19 +10,19 @@
 # Creates all prereq images (delphi_database, delphi_python) only if they don't
 # exist. If you need to rebuild a prereq, you're probably doing something
 # complicated, and can figure out the rebuild command on your own.
-# 
-# 
+#
+#
 # Commands:
-#            
+#
 #   web:     Stops currently-running delphi_web_epidata instances, if any.
 #            Rebuilds delphi_web_epidata image.
 #            Runs image in the background and pipes stdout to a log file.
-# 
+#
 #   db:      Stops currently-running delphi_database_epidata instances, if any.
 #            Rebuilds delphi_database_epidata image.
 #            Runs image in the background and pipes stdout to a log file.
 #            Blocks until database is ready to receive connections.
-# 
+#
 #   python:  Rebuilds delphi_web_python image. You shouldn't need to do this
 #            often; only if you are installing a new environment, or have
 #            made changes to delphi-epidata/dev/docker/python/Dockerfile.
@@ -35,7 +35,7 @@
 #
 #   clean:   Cleans up dangling Docker images.
 #
-# 
+#
 # Optional arguments:
 #   pdb=1         Drops you into debug mode upon test failure, if running tests.
 #   test=         Only runs tests in the directories provided here, e.g.
@@ -108,6 +108,7 @@ db:
 	@# Run the database
 	@docker run --rm -p 127.0.0.1:13306:3306 \
 		--network delphi-net --name delphi_database_epidata \
+                --cap-add=sys_nice \
 		delphi_database_epidata >$(LOG_DB) 2>&1 &
 
 	@# Block until DB is ready
@@ -130,7 +131,7 @@ py:
 all: db web py
 
 .PHONY=test
-test: 
+test:
 	@docker run -i --rm --network delphi-net \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \

devops/Dockerfile

@@ -3,13 +3,16 @@ WORKDIR /src
 COPY . /src
 RUN npm ci && npm run build
 
-FROM tiangolo/meinheld-gunicorn:python3.7
+FROM tiangolo/meinheld-gunicorn:python3.8
 LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epidata
 # use delphi's timezome
 RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
 
-COPY requirements.txt /app
-RUN pip install --no-cache-dir -r requirements.txt
+COPY requirements.txt /app/requirements_also.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
+# the file /tmp/requirements.txt is created in the parent docker definition.  (see:
+#   https://github.com/tiangolo/meinheld-gunicorn-docker/blob/master/docker-images/python3.8.dockerfile#L5 )
+# this combined requirements installation ensures all version constrants are accounted for.
 
 # disable python stdout buffering
 ENV PYTHONUNBUFFERED 1

integrations/acquisition/covidcast/delete_batch.csv

@@ -1,3 +1,4 @@
 geo_id,value,stderr,sample_size,issue,time_value,geo_type,signal,source
 d_nonlatest,0,0,0,1,0,geo,sig,src
-d_latest,   0,0,0,3,0,geo,sig,src
+d_latest,   0,0,0,3,0,geo,sig,src
+d_justone,  0,0,0,1,0,geo,sig,src

integrations/acquisition/covidcast/test_covidcast_meta_caching.py

@@ -36,13 +36,17 @@ def setUp(self):
         user='user',
         password='pass',
         host='delphi_database_epidata',
-        database='epidata')
+        database='covid')
     cur = cnx.cursor()
 
-    # clear the `covidcast` table
-    cur.execute('truncate table covidcast')
+    # clear all tables
+    cur.execute("truncate table epimetric_load")
+    cur.execute("truncate table epimetric_full")
+    cur.execute("truncate table epimetric_latest")
+    cur.execute("truncate table geo_dim")
+    cur.execute("truncate table signal_dim")
     # reset the `covidcast_meta_cache` table (it should always have one row)
-    cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = ""')
+    cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"')
     cnx.commit()
     cur.close()
 
@@ -67,30 +71,29 @@ def test_caching(self):
 
     # insert dummy data
     self.cur.execute(f'''
-      INSERT INTO
-        `covidcast` (`id`, `source`, `signal`, `time_type`, `geo_type`, 
-	      `time_value`, `geo_value`, `value_updated_timestamp`, 
-        `value`, `stderr`, `sample_size`, `direction_updated_timestamp`, 
-        `direction`, `issue`, `lag`, `is_latest_issue`, `is_wip`,`missing_value`,
-        `missing_stderr`,`missing_sample_size`) 
+      INSERT INTO `signal_dim` (`signal_key_id`, `source`, `signal`)
       VALUES
-        (0, 'src', 'sig', 'day', 'state', 20200422, 'pa',
-          123, 1, 2, 3, 456, 1, 20200422, 0, 1, False, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}),
-        (0, 'src', 'sig', 'day', 'state', 20200422, 'wa',
-          789, 1, 2, 3, 456, 1, 20200423, 1, 1, False, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING})
+        (42, 'src', 'sig');
+    ''')
+    self.cur.execute(f'''
+      INSERT INTO `geo_dim` (`geo_key_id`, `geo_type`, `geo_value`)
+      VALUES
+        (96, 'state', 'pa'), 
+        (97, 'state', 'wa');
     ''')
     self.cur.execute(f'''
       INSERT INTO
-        `covidcast` (`id`, `source`, `signal`, `time_type`, `geo_type`, 
-	      `time_value`, `geo_value`, `value_updated_timestamp`, 
-        `value`, `stderr`, `sample_size`, `direction_updated_timestamp`, 
-        `direction`, `issue`, `lag`, `is_latest_issue`, `is_wip`,`missing_value`,
-        `missing_stderr`,`missing_sample_size`) 
+        `epimetric_latest` (`epimetric_id`, `signal_key_id`, `geo_key_id`, `time_type`,
+	      `time_value`, `value_updated_timestamp`,
+        `value`, `stderr`, `sample_size`,
+        `issue`, `lag`, `missing_value`,
+        `missing_stderr`,`missing_sample_size`)
       VALUES
-        (100, 'src', 'wip_sig', 'day', 'state', 20200422, 'pa',
-          456, 4, 5, 6, 789, -1, 20200422, 0, 1, True, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING})
+        (15, 42, 96, 'day', 20200422,
+          123, 1, 2, 3, 20200422, 0, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}),
+        (16, 42, 97, 'day', 20200422,
+          789, 1, 2, 3, 20200423, 1, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING})
     ''')
-
     self.cnx.commit()
 
     # make sure the live utility is serving something sensible

integrations/acquisition/covidcast/test_csv_uploading.py

@@ -32,9 +32,18 @@ def setUp(self):
         user='user',
         password='pass',
         host='delphi_database_epidata',
-        database='epidata')
+        database='covid')
     cur = cnx.cursor()
-    cur.execute('truncate table covidcast')
+
+    # clear all tables
+    cur.execute("truncate table epimetric_load")
+    cur.execute("truncate table epimetric_full")
+    cur.execute("truncate table epimetric_latest")
+    cur.execute("truncate table geo_dim")
+    cur.execute("truncate table signal_dim")
+    # reset the `covidcast_meta_cache` table (it should always have one row)
+    cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"')
+
     cnx.commit()
     cur.close()
 
@@ -68,11 +77,12 @@ def apply_lag(expected_epidata):
     return expected_epidata
 
   def verify_timestamps_and_defaults(self):
-    self.cur.execute('select value_updated_timestamp, direction_updated_timestamp, direction from covidcast')
-    for value_updated_timestamp, direction_updated_timestamp, direction in self.cur:
+    self.cur.execute('''
+select value_updated_timestamp from epimetric_full
+UNION ALL
+select value_updated_timestamp from epimetric_latest''')
+    for (value_updated_timestamp,) in self.cur:
       self.assertGreater(value_updated_timestamp, 0)
-      self.assertEqual(direction_updated_timestamp, 0)
-      self.assertIsNone(direction)
 
   def test_uploading(self):
     """Scan, parse, upload, archive, serve, and fetch a covidcast signal."""
@@ -91,8 +101,6 @@ def test_uploading(self):
           log_file=log_file_directory +
           "output.log",
           data_dir=data_dir,
-          is_wip_override=False,
-          not_wip_override=False,
           specific_issue_date=False)
     uploader_column_rename = {"geo_id": "geo_value", "val": "value", "se": "stderr", "missing_val": "missing_value", "missing_se": "missing_stderr"}
 
@@ -217,41 +225,6 @@ def test_uploading(self):
       self.setUp()
 
 
-    with self.subTest("Valid wip"):
-      values = pd.DataFrame({
-        "geo_id": ["me", "nd", "wa"],
-        "val": [10.0, 20.0, 30.0],
-        "se": [0.01, 0.02, 0.03],
-        "sample_size": [100.0, 200.0, 300.0],
-        "missing_val": [Nans.NOT_MISSING] * 3,
-        "missing_se": [Nans.NOT_MISSING] * 3,
-        "missing_sample_size": [Nans.NOT_MISSING] * 3
-      })
-      signal_name = "wip_prototype"
-      values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)
-
-      # upload CSVs
-      main(args)
-      response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
-
-      expected_values = pd.concat([values, pd.DataFrame({
-        "time_value": [20200419] * 3,
-        "signal": [signal_name] * 3,
-        "direction": [None] * 3
-      })], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records")
-      expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'}
-
-      self.assertEqual(response, expected_response)
-      self.verify_timestamps_and_defaults()
-
-      # Verify that files were archived
-      path = data_dir + f'/archive/successful/src-name/20200419_state_wip_prototype.csv.gz'
-      self.assertIsNotNone(os.stat(path))
-
-      self.tearDown()
-      self.setUp()
-
-
     with self.subTest("Valid signal with name length 32<x<64"):
       values = pd.DataFrame({
         "geo_id": ["pa"],
@@ -262,7 +235,7 @@ def test_uploading(self):
         "missing_se": [Nans.NOT_MISSING],
         "missing_sample_size": [Nans.NOT_MISSING]
       })
-      signal_name = "wip_really_long_name_that_will_be_accepted"
+      signal_name = "really_long_name_that_will_be_accepted"
       values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)
 
       # upload CSVs
@@ -293,7 +266,7 @@ def test_uploading(self):
         "missing_se": [Nans.NOT_MISSING],
         "missing_sample_size": [Nans.NOT_MISSING]
       })
-      signal_name = "wip_really_long_name_that_will_get_truncated_lorem_ipsum_dolor_sit_amet"
+      signal_name = "really_long_name_that_will_get_truncated_lorem_ipsum_dolor_sit_amet"
       values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)
 
       # upload CSVs