Merge pull request #685 from cmu-delphi/release/delphi-epidata-0.2.8

Release Delphi Epidata 0.2.8

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.7
+current_version = 0.2.8
 commit = False
 tag = False
 

src/acquisition/covidcast/database.py

@@ -257,6 +257,7 @@ def compute_covidcast_meta(self, table_name='covidcast', use_index=True):
     n_threads = max(1, cpu_count()*9//10) # aka number of concurrent db connections, which [sh|c]ould be ~<= 90% of the #cores available to SQL server
     # NOTE: this may present a small problem if this job runs on different hardware than the db,
     #       but we should not run into that issue in prod.
+    logger.info(f"using {n_threads} workers")
 
     srcsigs = Queue() # multi-consumer threadsafe!
 
@@ -305,22 +306,24 @@ def compute_covidcast_meta(self, table_name='covidcast', use_index=True):
     meta_lock = threading.Lock()
 
     def worker():
-      logger.info("starting thread: " + threading.current_thread().name)
+      name = threading.current_thread().name
+      logger.info("starting thread", thread=name)
       #  set up new db connection for thread
       worker_dbc = Database()
       worker_dbc.connect(connector_impl=self._connector_impl)
       w_cursor = worker_dbc._cursor
       try:
         while True:
           (source, signal) = srcsigs.get_nowait() # this will throw the Empty caught below
+          logger.info("starting pair", thread=name, pair=f"({source}, {signal})")
           w_cursor.execute(inner_sql, (source, signal))
           with meta_lock:
             meta.extend(list(
               dict(zip(w_cursor.column_names, x)) for x in w_cursor
             ))
           srcsigs.task_done()
       except Empty:
-        logger.info("no jobs left, thread terminating: " + threading.current_thread().name)
+        logger.info("no jobs left, thread terminating", thread=name)
       finally:
         worker_dbc.disconnect(False) # cleanup
 
@@ -334,7 +337,7 @@ def worker():
     logger.info("jobs complete")
     for t in threads:
       t.join()
-    logger.error("threads terminated")
+    logger.info("all threads terminated")
 
     # sort the metadata because threaded workers dgaf
     sorting_fields = "data_source signal time_type geo_type".split()

src/client/delphi_epidata.R

@@ -15,7 +15,7 @@ Epidata <- (function() {
   # API base url
   BASE_URL <- 'https://delphi.cmu.edu/epidata/api.php'
 
-  client_version <- '0.2.7'
+  client_version <- '0.2.8'
 
   # Helper function to cast values and/or ranges to strings
   .listitem <- function(value) {

src/client/delphi_epidata.js

@@ -22,7 +22,7 @@
   }
 })(this, function (exports, fetchImpl, jQuery) {
   const BASE_URL = "https://delphi.cmu.edu/epidata/";
-  const client_version = "0.2.7";
+  const client_version = "0.2.8";
 
   // Helper function to cast values and/or ranges to strings
   function _listitem(value) {

src/client/packaging/npm/package-lock.json

@@ -2,7 +2,7 @@
   "name": "delphi_epidata",
   "description": "Delphi Epidata API Client",
   "authors": "Delphi Group",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "license": "MIT",
   "homepage": "https://github.com/cmu-delphi/delphi-epidata",
   "bugs": {

src/client/packaging/npm/package.json

@@ -1,4 +1,4 @@
 from .delphi_epidata import Epidata
 
 name = 'delphi_epidata'
-__version__ = '0.2.7'
+__version__ = '0.2.8'

src/client/packaging/pypi/delphi_epidata/__init__.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="delphi_epidata",
-    version="0.2.7",
+    version="0.2.8",
     author="David Farrow",
     author_email="[email protected]",
     description="A programmatic interface to Delphi's Epidata API.",

src/client/packaging/pypi/setup.py

@@ -5,7 +5,7 @@
 
 load_dotenv()
 
-VERSION = "0.2.7"
+VERSION = "0.2.8"
 
 MAX_RESULTS = int(10e6)
 MAX_COMPATIBILITY_RESULTS = int(3650)

src/server/_config.py

@@ -5,8 +5,9 @@
 
 from flask import request
 
+
 from ._exceptions import ValidationFailedException
-from .utils import days_in_range, weeks_in_range
+from .utils import days_in_range, weeks_in_range, guess_time_value_is_day
 
 
 def _parse_common_multi_arg(key: str) -> List[Tuple[str, Union[bool, Sequence[str]]]]:
@@ -109,6 +110,15 @@ class TimePair:
     time_type: str
     time_values: Union[bool, Sequence[Union[int, Tuple[int, int]]]]
 
+    @property
+    def is_week(self) -> bool:
+        return self.time_type == 'week'
+
+    @property
+    def is_day(self) -> bool:
+        return self.time_type != 'week'
+
+
     def count(self) -> float:
         """
         returns the count of items in this pair
@@ -225,3 +235,45 @@ def parse_day_arg(key: str) -> int:
     if not isinstance(r, int):
         raise ValidationFailedException(f"{key} must match YYYYMMDD or YYYY-MM-DD")
     return r
+
+def parse_week_arg(key: str) -> int:
+    v = request.values.get(key)
+    if not v:
+        raise ValidationFailedException(f"{key} param is required")
+    r = parse_week_value(v)
+    if not isinstance(r, int):
+        raise ValidationFailedException(f"{key} must match YYYYWW")
+    return r
+
+
+def parse_week_range_arg(key: str) -> Tuple[int, int]:
+    v = request.values.get(key)
+    if not v:
+        raise ValidationFailedException(f"{key} param is required")
+    r = parse_week_value(v)
+    if not isinstance(r, tuple):
+        raise ValidationFailedException(f"{key} must match YYYYWW-YYYYWW")
+    return r
+
+def parse_day_or_week_arg(key: str, default_value: Optional[int] = None) -> Tuple[int, bool]:
+    v = request.values.get(key)
+    if not v:
+        if default_value is not None:
+            return default_value, guess_time_value_is_day(default_value)
+        raise ValidationFailedException(f"{key} param is required")
+    # format is either YYYY-MM-DD or YYYYMMDD or YYYYMM
+    is_week = len(v) == 6
+    if is_week:
+        return parse_week_arg(key), False
+    return parse_day_arg(key), True
+
+def parse_day_or_week_range_arg(key: str) -> Tuple[Tuple[int, int], bool]:
+    v = request.values.get(key)
+    if not v:
+        raise ValidationFailedException(f"{key} param is required")
+    # format is either YYYY-MM-DD--YYYY-MM-DD or YYYYMMDD-YYYYMMDD or YYYYMM-YYYYMM
+    # so if the first before the - has length 6, it must be a week
+    is_week = len(v.split('-', 2)[0]) == 6
+    if is_week:
+        return parse_week_range_arg(key), False
+    return parse_day_range_arg(key), True