Merge remote-tracking branch 'origin/dev' into sgratzl/docker2

Author: sgratzl

.github/workflows/ci.yaml

@@ -24,7 +24,7 @@ jobs:
             ${{ runner.os }}-conda6-
       - uses: conda-incubator/setup-miniconda@v2
         with:
-          python-version: 3.7
+          python-version: 3.9
           mamba-version: "*"
           channels: conda-forge,defaults
           channel-priority: true

content/blog/2020-08-26-fb-survey.Rmd

@@ -149,22 +149,23 @@ df_in = covidcast_signal("jhu-csse", "confirmed_7dav_incidence_prop",
 df_fb_avg = df_fb %>% group_by(geo_value) %>% summarize(value = mean(value))
 df_in_avg = df_in %>% group_by(geo_value) %>% summarize(value = mean(value))
 
-# Set a bunch of fields so that the data frames know how to plot themselves
-df_fb_avg$time_value = df_in_avg$time_value = start_day
-df_fb_avg$issue = df_in_avg$issue = start_day
-attributes(df_fb_avg)$metadata$geo_type = "state"
-attributes(df_in_avg)$metadata$geo_type = "state"
-class(df_fb_avg) = c("covidcast_signal", class(df_fb_avg))
-class(df_in_avg) = c("covidcast_signal", class(df_in_avg))
+# Turn these into covidcast_signal data frames that can be mapped
+df_fb_avg = df_fb_avg %>%
+  mutate(time_value = start_day) %>%
+  as.covidcast_signal(signal = "fb_avg", geo_type = "state")
+
+df_in_avg = df_in_avg %>%
+  mutate(time_value = start_day) %>%
+  as.covidcast_signal(signal = "in_avg", geo_type = "state")
 
 # Plot choropleth maps, using the covidcast plotting functionality
 subtitle = paste("Averaged over", start_day, "to", end_day)
 p1 = plot(df_fb_avg,
           title = "% of people with COVID symptoms, based on Facebook surveys",
-          range=c(0, 1), choro_params = list(subtitle = subtitle))
+          range = c(0, 1), choro_params = list(subtitle = subtitle))
 p2 = plot(df_in_avg,
           title = "Daily new confirmed COVID-19 cases per 100,000 people",
-          range=c(0, 30), choro_params = list(subtitle = subtitle))
+          range = c(0, 30), choro_params = list(subtitle = subtitle))
 grid.arrange(p1, p2, nrow = 1)
 ```
 
@@ -539,12 +540,13 @@ or we didn't have enough data from the surveys in
 order to estimate % CLI and % CLI-in-community signals there.
 
 ```{r, fig.width = 10, fig.height = 4, out.extra = 'class="wide-figure"'}
-# Set a bunch of fields so that the data frames know how to plot themselves
-df_cor1$time_value = df_cor2$time_value = start_day
-df_cor1$issue = df_cor2$issue = start_day
-attributes(df_cor1)$metadata$geo_type = "county"
-attributes(df_cor2)$metadata$geo_type = "county"
-class(df_cor1) = class(df_cor2) = c("covidcast_signal", "data.frame")
+# Use as.covidcast_signal to turn these into mappable signals
+df_cor1 = df_cor1 %>%
+  mutate(time_value = start_day) %>%
+  as.covidcast_signal(signal = "cor1")
+df_cor2 = df_cor2 %>%
+  mutate(time_value = start_day) %>%
+  as.covidcast_signal(signal = "cor1")
 
 # Plot choropleth maps, using the covidcast plotting functionality
 p1 = plot(df_cor1, title = "Correlation between % CLI and case rates",

content/blog/2020-08-26-fb-survey.html

@@ -36,7 +36,7 @@
   survey data. The COVIDcast API, which serves these estimates and is updated
   daily, is a much larger effort run by the Delphi group, and our entire
   engineering team is owed a lot of credit here. Ryan came up with the
-  idea of running the surveys, and worked with Facebook to make this a reality.
+  idea of running the surveys, and worked with Facebook to make this a reality. 
   On the Facebook side, Curtiss Cobb and Jonathan McKay played big roles. The
   University of Maryland team, including Adrianne Bradford and Samantha Chiu and
   led by Frauke Kreuter, made many contributions to the survey design.
@@ -47,7 +47,6 @@
     toc: true
 ---
 
-<script src="/rmarkdown-libs/header-attrs/header-attrs.js"></script>
 <link href="/rmarkdown-libs/anchor-sections/anchor-sections.css" rel="stylesheet" />
 <script src="/rmarkdown-libs/anchor-sections/anchor-sections.js"></script>
 
@@ -57,8 +56,7 @@
 <li><a href="#why-run-these-surveys">Why Run These Surveys?</a></li>
 <li><a href="#whats-in-the-survey">What’s in the Survey?</a></li>
 <li><a href="#some-interesting-examples">Some Interesting Examples</a></li>
-<li><a href="#basic-correlation-analysis">Basic Correlation Analysis</a>
-<ul>
+<li><a href="#basic-correlation-analysis">Basic Correlation Analysis</a><ul>
 <li><a href="#correlations-sliced-by-time">Correlations Sliced by Time</a></li>
 <li><a href="#correlations-sliced-by-county">Correlations Sliced by County</a></li>
 </ul></li>
@@ -156,22 +154,23 @@ <h2>Short Background</h2>
 df_fb_avg = df_fb %&gt;% group_by(geo_value) %&gt;% summarize(value = mean(value))
 df_in_avg = df_in %&gt;% group_by(geo_value) %&gt;% summarize(value = mean(value))
 
-# Set a bunch of fields so that the data frames know how to plot themselves
-df_fb_avg$time_value = df_in_avg$time_value = start_day
-df_fb_avg$issue = df_in_avg$issue = start_day
-attributes(df_fb_avg)$metadata$geo_type = &quot;state&quot;
-attributes(df_in_avg)$metadata$geo_type = &quot;state&quot;
-class(df_fb_avg) = c(&quot;covidcast_signal&quot;, class(df_fb_avg))
-class(df_in_avg) = c(&quot;covidcast_signal&quot;, class(df_in_avg))
+# Turn these into covidcast_signal data frames that can be mapped
+df_fb_avg = df_fb_avg %&gt;%
+  mutate(time_value = start_day) %&gt;%
+  as.covidcast_signal(signal = &quot;fb_avg&quot;, geo_type = &quot;state&quot;)
+
+df_in_avg = df_in_avg %&gt;%
+  mutate(time_value = start_day) %&gt;%
+  as.covidcast_signal(signal = &quot;in_avg&quot;, geo_type = &quot;state&quot;)
 
 # Plot choropleth maps, using the covidcast plotting functionality
 subtitle = paste(&quot;Averaged over&quot;, start_day, &quot;to&quot;, end_day)
 p1 = plot(df_fb_avg,
           title = &quot;% of people with COVID symptoms, based on Facebook surveys&quot;,
-          range=c(0, 1), choro_params = list(subtitle = subtitle))
+          range = c(0, 1), choro_params = list(subtitle = subtitle))
 p2 = plot(df_in_avg,
           title = &quot;Daily new confirmed COVID-19 cases per 100,000 people&quot;,
-          range=c(0, 30), choro_params = list(subtitle = subtitle))
+          range = c(0, 30), choro_params = list(subtitle = subtitle))
 grid.arrange(p1, p2, nrow = 1)</code></pre>
 <p><img src="/blog/2020-08-26-fb-survey_files/figure-html/unnamed-chunk-2-1.svg" width="960" class="wide-figure" /></p>
 <p>We generated these plots using our <a href="https://cmu-delphi.github.io/covidcast/covidcastR/">covidcast R
@@ -517,12 +516,13 @@ <h3>Correlations Sliced by County</h3>
 either that county had below 500 cumulative COVID-19 cases,
 or we didn’t have enough data from the surveys in
 order to estimate % CLI and % CLI-in-community signals there.</p>
-<pre class="r"><code># Set a bunch of fields so that the data frames know how to plot themselves
-df_cor1$time_value = df_cor2$time_value = start_day
-df_cor1$issue = df_cor2$issue = start_day
-attributes(df_cor1)$metadata$geo_type = &quot;county&quot;
-attributes(df_cor2)$metadata$geo_type = &quot;county&quot;
-class(df_cor1) = class(df_cor2) = c(&quot;covidcast_signal&quot;, &quot;data.frame&quot;)
+<pre class="r"><code># Use as.covidcast_signal to turn these into mappable signals
+df_cor1 = df_cor1 %&gt;%
+  mutate(time_value = start_day) %&gt;%
+  as.covidcast_signal(signal = &quot;cor1&quot;)
+df_cor2 = df_cor2 %&gt;%
+  mutate(time_value = start_day) %&gt;%
+  as.covidcast_signal(signal = &quot;cor1&quot;)
 
 # Plot choropleth maps, using the covidcast plotting functionality
 p1 = plot(df_cor1, title = &quot;Correlation between % CLI and case rates&quot;,

content/blog/2020-10-14-dv-signal.Rmd

@@ -89,13 +89,13 @@ df_in = covidcast_signal("usa-facts", "confirmed_7dav_incidence_prop",
 df_dv_avg = df_dv %>% group_by(geo_value) %>% summarize(value = mean(value))
 df_in_avg = df_in %>% group_by(geo_value) %>% summarize(value = mean(value))
 
-# Set a bunch of fields so that the data frames know how to plot themselves
-df_dv_avg$time_value = df_in_avg$time_value = start_day
-df_dv_avg$issue = df_in_avg$issue = start_day
-attributes(df_dv_avg)$metadata$geo_type = "state"
-attributes(df_in_avg)$metadata$geo_type = "state"
-class(df_dv_avg) = c("covidcast_signal", class(df_dv_avg))
-class(df_in_avg) = c("covidcast_signal", class(df_in_avg))
+# Use as.covidcast_signal to turn these into covidcast data frames
+df_dv_avg <- df_dv_avg %>%
+  mutate(time_value = start_day) %>%
+  as.covidcast_signal(signal = "dv_avg", geo_type = "state")
+df_in_avg <- df_in_avg %>%
+  mutate(time_value = start_day) %>%
+  as.covidcast_signal(signal = "in_avg", geo_type = "state")
 
 # Plot choropleth maps, using the covidcast plotting functionality
 subtitle = paste("Averaged over", start_day, "to", end_day)
@@ -337,8 +337,12 @@ lm_dv_cases_df = dv_cases_df %>%
   ungroup()
 
 df_cor_by_time_adj = covidcast_cor(
-  dv_cases_df %>% mutate(value = dv*slope_6wk + int_6wk, issue = time_value),
-  dv_cases_df %>% mutate(value = cases, issue = time_value),
+  dv_cases_df %>%
+    mutate(value = dv*slope_6wk + int_6wk, issue = time_value) %>%
+    as.covidcast_signal(signal = "dv_cases"),
+  dv_cases_df %>%
+    mutate(value = cases, issue = time_value) %>%
+    as.covidcast_signal(signal = "dv_cases"),
   by = "time_value",
   method = "spearman")
 

content/blog/2020-10-14-dv-signal.html

@@ -17,8 +17,8 @@
 summary: |
   In previous posts, we discussed our massive ongoing symptom surveys that have reached over 12 million people in the U.S. since April 2020, in partnership with Facebook and Google. Another one of our major data initiatives is based on partnerships with healthcare systems, granting us access to various aggregate statistics from hospital records and insurance claims covering 10-15% of the United States population. From these data, we can extract informative indicators that can be early indicators of COVID activity. This post focuses on one indicator in particular, based on outpatient visits, and demonstrates both the challenges and promises associated with medical records data.
 acknowledgements: |
-  Maria Jahja contributed immensely to every stage of this project, from determining which ICD codes to use to the final implementation of the indicator.
-  Aaron Rumack devised the weekday adjustment and analyzed the performance of the DV indicator.
+  Maria Jahja contributed immensely to every stage of this project, from determining which ICD codes to use to the final implementation of the indicator. 
+  Aaron Rumack devised the weekday adjustment and analyzed the performance of the DV indicator. 
   Roni Rosenfeld worked closely with our health systems partners to get access to the data and provided domain knowledge to ensure that the data was useful. Both Roni and Ryan Tibshirani provided helpful suggestions and insights towards the methodology and analysis.
 
 output:
@@ -28,7 +28,6 @@
     toc: true
 ---
 
-<script src="/rmarkdown-libs/header-attrs/header-attrs.js"></script>
 <link href="/rmarkdown-libs/anchor-sections/anchor-sections.css" rel="stylesheet" />
 <script src="/rmarkdown-libs/anchor-sections/anchor-sections.js"></script>
 
@@ -88,13 +87,13 @@ <h2>The Doctor Visits Indicator</h2>
 df_dv_avg = df_dv %&gt;% group_by(geo_value) %&gt;% summarize(value = mean(value))
 df_in_avg = df_in %&gt;% group_by(geo_value) %&gt;% summarize(value = mean(value))
 
-# Set a bunch of fields so that the data frames know how to plot themselves
-df_dv_avg$time_value = df_in_avg$time_value = start_day
-df_dv_avg$issue = df_in_avg$issue = start_day
-attributes(df_dv_avg)$metadata$geo_type = &quot;state&quot;
-attributes(df_in_avg)$metadata$geo_type = &quot;state&quot;
-class(df_dv_avg) = c(&quot;covidcast_signal&quot;, class(df_dv_avg))
-class(df_in_avg) = c(&quot;covidcast_signal&quot;, class(df_in_avg))
+# Use as.covidcast_signal to turn these into covidcast data frames
+df_dv_avg &lt;- df_dv_avg %&gt;%
+  mutate(time_value = start_day) %&gt;%
+  as.covidcast_signal(signal = &quot;dv_avg&quot;, geo_type = &quot;state&quot;)
+df_in_avg &lt;- df_in_avg %&gt;%
+  mutate(time_value = start_day) %&gt;%
+  as.covidcast_signal(signal = &quot;in_avg&quot;, geo_type = &quot;state&quot;)
 
 # Plot choropleth maps, using the covidcast plotting functionality
 subtitle = paste(&quot;Averaged over&quot;, start_day, &quot;to&quot;, end_day)
@@ -317,8 +316,12 @@ <h2>Spatial Heterogeneity</h2>
   ungroup()
 
 df_cor_by_time_adj = covidcast_cor(
-  dv_cases_df %&gt;% mutate(value = dv*slope_6wk + int_6wk, issue = time_value),
-  dv_cases_df %&gt;% mutate(value = cases, issue = time_value),
+  dv_cases_df %&gt;%
+    mutate(value = dv*slope_6wk + int_6wk, issue = time_value) %&gt;%
+    as.covidcast_signal(signal = &quot;dv_cases&quot;),
+  dv_cases_df %&gt;%
+    mutate(value = cases, issue = time_value) %&gt;%
+    as.covidcast_signal(signal = &quot;dv_cases&quot;),
   by = &quot;time_value&quot;,
   method = &quot;spearman&quot;)
 

content/covidcast/release-log/headless/v1.12.4.md

@@ -0,0 +1,14 @@
+---
+title: v1.12.4
+date: 2021-01-19
+---
+
+This release improves the UX of the survey dashboard for both desktop and mobile experience.
+
+#### New Features
+
+- [#724](https://github.com/cmu-delphi/www-covidcast/pull/724), [#717](https://github.com/cmu-delphi/www-covidcast/pull/717) Survey dashboards improvements
+
+#### Bug-fixes
+
+- [#719](https://github.com/cmu-delphi/www-covidcast/pull/719) (re: [#718](https://github.com/cmu-delphi/www-covidcast/issues/718)) fix Export Data bug

devops/r.dockerfile

@@ -1,8 +1,10 @@
 # docker image for setting up an R environment
 FROM continuumio/miniconda
 
+RUN conda install --name base --channel conda-forge mamba=*
+
 ADD ./environment.yml .
-RUN conda env create -f environment.yml
+RUN mamba env create -f environment.yml
 
 ADD ./dependencies.R .
 

environment.yml

@@ -3,9 +3,9 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - gdal=3.0.2
-  - python=3.7.9
-  - pip=20.2.4
+  - gdal
+  - python=3.9.1
+  - pip
   - r-base=3.6.3
   - glib
   - mscorefonts
@@ -32,7 +32,8 @@ dependencies:
   - r-isoband
   - r-ggrepel
   - r-plotly
-  - r-usmap
+  - r-sf
+  - r-lubridate
   - pandoc
   - fiona
   - geopandas
@@ -41,5 +42,6 @@ dependencies:
   - pillow
   - pyproj
   - matplotlib
+  - udunits2
   - pip:
-      - covidcast==0.1.2
+      - covidcast==0.1.3

package-lock.json

@@ -1,13 +1,13 @@
 {
   "name": "www-main",
-  "version": "1.12.3",
+  "version": "1.12.4",
   "private": true,
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.15.1",
     "highlight.js": "^10.5.0",
     "katex": "^0.12.0",
     "uikit": "^3.6.5",
-    "www-covidcast": "https://github.com/cmu-delphi/www-covidcast/releases/download/v1.12.3/www-covidcast-1.12.3.tgz"
+    "www-covidcast": "https://github.com/cmu-delphi/www-covidcast/releases/download/v1.12.4/www-covidcast-1.12.4.tgz"
   },
   "devDependencies": {
     "hugo-bin": "^0.67.1",