From b38e4a86fc9064cb8015ac7a3acbaa9f1462272c Mon Sep 17 00:00:00 2001
From: Antoine Roy-Gobeil <antoine@plot.ly>
Date: Wed, 6 Mar 2019 21:10:35 -0500
Subject: [PATCH 1/5] test-bundle in a separate job, enable parallelism

---
 .circleci/config.yml              | 17 +++++++++++++++++
 .circleci/test.sh                 | 14 ++++++++++----
 test/jasmine/tests/sankey_test.js |  2 +-
 3 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 33d2638b085..b214d550e51 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -40,6 +40,7 @@ jobs:
     docker:
       # need '-browsers' version to test in real (xvfb-wrapped) browsers
       - image: circleci/node:10.9.0-browsers
+    parallelism: 2
     working_directory: ~/plotly.js
     steps:
       - attach_workspace:
@@ -52,6 +53,7 @@ jobs:
     docker:
       # need '-browsers' version to test in real (xvfb-wrapped) browsers
       - image: circleci/node:10.9.0-browsers
+    parallelism: 2
     working_directory: ~/plotly.js
     steps:
       - attach_workspace:
@@ -121,6 +123,18 @@ jobs:
           name: Run syntax tests
           command: ./.circleci/test.sh syntax
 
+  test-bundle:
+    docker:
+      # need '-browsers' version to test in real (xvfb-wrapped) browsers
+      - image: circleci/node:10.9.0-browsers
+    working_directory: ~/plotly.js
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          name: Run test-bundle
+          command: ./.circleci/test.sh bundle
+
   publish:
     docker:
       - image: circleci/node:10.9.0
@@ -157,6 +171,9 @@ workflows:
   build-and-test:
     jobs:
       - build
+      - test-bundle:
+          requires:
+            - build
       - test-jasmine:
           requires:
             - build
diff --git a/.circleci/test.sh b/.circleci/test.sh
index 62fd478837f..d090dfa2b82 100755
--- a/.circleci/test.sh
+++ b/.circleci/test.sh
@@ -44,8 +44,8 @@ case $1 in
     jasmine)
         set_tz
 
-        npm run test-jasmine -- --skip-tags=gl,noCI,flaky || EXIT_STATE=$?
-        npm run test-bundle || EXIT_STATE=$?
+        SUITE=$(circleci tests glob "$ROOT/test/jasmine/tests/*" | circleci tests split)
+        npm run test-jasmine -- $SUITE --skip-tags=gl,noCI,flaky || EXIT_STATE=$?
 
         exit $EXIT_STATE
         ;;
@@ -53,7 +53,7 @@ case $1 in
     jasmine2)
         set_tz
 
-        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl))
+        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl | circleci tests split))
 
         for s in ${SHARDS[@]}; do
             retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI
@@ -65,7 +65,7 @@ case $1 in
     jasmine3)
         set_tz
 
-        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=flaky))
+        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=flaky | circleci tests split))
 
         for s in ${SHARDS[@]}; do
             retry npm run test-jasmine -- "$s" --tags=flaky --skip-tags=noCI
@@ -85,6 +85,12 @@ case $1 in
         exit $EXIT_STATE
         ;;
 
+    bundle)
+        set_tz
+        npm run test-bundle || EXIT_STATE=$?
+        exit $EXIT_STATE
+        ;;
+
     syntax)
         npm run lint        || EXIT_STATE=$?
         npm run test-syntax || EXIT_STATE=$?
diff --git a/test/jasmine/tests/sankey_test.js b/test/jasmine/tests/sankey_test.js
index 69ec5616e73..81eb93ae5c4 100644
--- a/test/jasmine/tests/sankey_test.js
+++ b/test/jasmine/tests/sankey_test.js
@@ -997,7 +997,7 @@ describe('sankey tests', function() {
         });
 
         ['node', 'link'].forEach(function(obj) {
-            it('should not output hover/unhover event data when ' + obj + '.hoverinfo is skip', function(done) {
+            it('@flaky should not output hover/unhover event data when ' + obj + '.hoverinfo is skip', function(done) {
                 var fig = Lib.extendDeep({}, mock);
 
                 Plotly.plot(gd, fig)

From 830d0a9feea72f0ea53fea53b277e31ce6bddf85 Mon Sep 17 00:00:00 2001
From: Antoine Roy-Gobeil <antoine@plot.ly>
Date: Wed, 13 Mar 2019 15:30:23 -0400
Subject: [PATCH 2/5] circleci: run test-image in parallel

---
 .circleci/config.yml              |  1 +
 .circleci/test.sh                 |  3 +-
 test/image/compare_pixels_test.js | 65 +++++++++++++++++++++----------
 3 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b214d550e51..cf506a19349 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -77,6 +77,7 @@ jobs:
   test-image:
     docker:
       - image: plotly/testbed:latest
+    parallelism: 2
     working_directory: /var/www/streambed/image_server/plotly.js/
     steps:
       - attach_workspace:
diff --git a/.circleci/test.sh b/.circleci/test.sh
index d090dfa2b82..803d103750c 100755
--- a/.circleci/test.sh
+++ b/.circleci/test.sh
@@ -75,7 +75,8 @@ case $1 in
         ;;
 
     image)
-        npm run test-image      || EXIT_STATE=$?
+        SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | circleci tests split)
+        npm run test-image -- $SUITE --filter || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 
diff --git a/test/image/compare_pixels_test.js b/test/image/compare_pixels_test.js
index e5a5a1f43f1..7776591ab4f 100644
--- a/test/image/compare_pixels_test.js
+++ b/test/image/compare_pixels_test.js
@@ -1,4 +1,5 @@
 var fs = require('fs');
+var minimist = require('minimist');
 
 var common = require('../../tasks/util/common');
 var getMockList = require('./assets/get_mock_list');
@@ -48,37 +49,61 @@ var QUEUE_WAIT = 10;
  *      npm run test-image -- gl3d_* --queue
  */
 
-var pattern = process.argv[2];
-var mockList = getMockList(pattern);
-var isInQueue = (process.argv[3] === '--queue');
+var argv = minimist(process.argv.slice(2), {boolean: ['queue', 'filter' ]});
+var isInQueue = argv.queue;
+var filter = argv.filter;
 
-if(mockList.length === 0) {
-    throw new Error('No mocks found with pattern ' + pattern);
+var allMock = false;
+// If no pattern is provided, all mocks are compared
+if(argv._.length === 0) {
+    allMock = true;
+    argv._.push('');
 }
 
-// filter out untestable mocks if no pattern is specified
-if(!pattern) {
-    console.log('Filtering out untestable mocks:');
-    mockList = mockList.filter(untestableFilter);
-    console.log('\n');
-}
+// Build list of mocks to compare
+var allMockList = [];
+argv._.forEach(function(pattern) {
+    var mockList = getMockList(pattern);
 
-// gl2d have limited image-test support
-if(pattern === 'gl2d_*') {
-    if(!isInQueue) {
-        console.log('WARN: Running gl2d image tests in batch may lead to unwanted results\n');
+    if(mockList.length === 0) {
+        throw new Error('No mocks found with pattern ' + pattern);
+    }
+
+    // gl2d have limited image-test support
+    if(pattern === 'gl2d_*') {
+        if(!isInQueue) {
+            console.log('WARN: Running gl2d image tests in batch may lead to unwanted results\n');
+        }
+        console.log('\nSorting gl2d mocks to avoid gl-shader conflicts');
+        sortGl2dMockList(mockList);
+        console.log('');
     }
-    console.log('\nSorting gl2d mocks to avoid gl-shader conflicts');
-    sortGl2dMockList(mockList);
-    console.log('');
+
+    allMockList = allMockList.concat(mockList);
+});
+
+// To get rid of duplicates
+Array.prototype.unique = function() {
+    return this.filter(function(value, index, self) {
+        return self.indexOf(value) === index;
+    });
+};
+allMockList = allMockList.unique();
+
+// filter out untestable mocks if no pattern is specified (ie. we're testing all mocks)
+// or if flag '--filter' is provided
+if(allMock || filter) {
+    console.log('Filtering out untestable mocks:');
+    allMockList = allMockList.filter(untestableFilter);
+    console.log('\n');
 }
 
 // main
 if(isInQueue) {
-    runInQueue(mockList);
+    runInQueue(allMockList);
 }
 else {
-    runInBatch(mockList);
+    runInBatch(allMockList);
 }
 
 /* Test cases:

From 609cdf89bf2e22f7ad9039e3b882528310a3e08e Mon Sep 17 00:00:00 2001
From: Antoine Roy-Gobeil <antoine@plot.ly>
Date: Mon, 25 Mar 2019 18:45:44 -0400
Subject: [PATCH 3/5] circleci: :hocho: special handling of `gl2d` mocks

---
 .circleci/config.yml              |  2 +-
 .circleci/test.sh                 |  1 -
 package.json                      |  1 -
 test/image/compare_pixels_test.js | 40 +------------------------------
 4 files changed, 2 insertions(+), 42 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cf506a19349..2630b369840 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -77,7 +77,7 @@ jobs:
   test-image:
     docker:
       - image: plotly/testbed:latest
-    parallelism: 2
+    parallelism: 4
     working_directory: /var/www/streambed/image_server/plotly.js/
     steps:
       - attach_workspace:
diff --git a/.circleci/test.sh b/.circleci/test.sh
index 803d103750c..95eee371a92 100755
--- a/.circleci/test.sh
+++ b/.circleci/test.sh
@@ -82,7 +82,6 @@ case $1 in
 
     image2)
         npm run test-export     || EXIT_STATE=$?
-        npm run test-image-gl2d || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 
diff --git a/package.json b/package.json
index 034b53eecff..7bf057fd941 100644
--- a/package.json
+++ b/package.json
@@ -35,7 +35,6 @@
     "pretest": "node tasks/pretest.js",
     "test-jasmine": "karma start test/jasmine/karma.conf.js",
     "test-image": "node tasks/test_image.js",
-    "test-image-gl2d": "node tasks/test_image.js gl2d_* --queue",
     "test-export": "node tasks/test_export.js",
     "test-syntax": "node tasks/test_syntax.js && npm run find-strings -- --no-output",
     "test-bundle": "node tasks/test_bundle.js",
diff --git a/test/image/compare_pixels_test.js b/test/image/compare_pixels_test.js
index 7776591ab4f..721af9e8f86 100644
--- a/test/image/compare_pixels_test.js
+++ b/test/image/compare_pixels_test.js
@@ -69,16 +69,6 @@ argv._.forEach(function(pattern) {
         throw new Error('No mocks found with pattern ' + pattern);
     }
 
-    // gl2d have limited image-test support
-    if(pattern === 'gl2d_*') {
-        if(!isInQueue) {
-            console.log('WARN: Running gl2d image tests in batch may lead to unwanted results\n');
-        }
-        console.log('\nSorting gl2d mocks to avoid gl-shader conflicts');
-        sortGl2dMockList(mockList);
-        console.log('');
-    }
-
     allMockList = allMockList.concat(mockList);
 });
 
@@ -120,7 +110,7 @@ else {
 function untestableFilter(mockName) {
     var cond = !(
         mockName === 'font-wishlist' ||
-        mockName.indexOf('gl2d_') !== -1 ||
+        // mockName.indexOf('gl2d_') !== -1 ||
         mockName.indexOf('mapbox_') !== -1
     );
 
@@ -129,34 +119,6 @@ function untestableFilter(mockName) {
     return cond;
 }
 
-/* gl2d pointcloud and other non-regl gl2d mock(s)
- * must be tested first on in order to work;
- * sort them here.
- *
- * gl-shader appears to conflict with regl.
- * We suspect that the lone gl context on CircleCI is
- * having issues with dealing with the two different
- * program binding algorithm.
- *
- * The problem will be solved by switching all our
- * WebGL-based trace types to regl.
- *
- * More info here:
- * https://github.com/plotly/plotly.js/pull/1037
- */
-function sortGl2dMockList(mockList) {
-    var mockNames = ['gl2d_pointcloud-basic', 'gl2d_heatmapgl'];
-    var pos = 0;
-
-    mockNames.forEach(function(m) {
-        var ind = mockList.indexOf(m);
-        var tmp = mockList[pos];
-        mockList[pos] = m;
-        mockList[ind] = tmp;
-        pos++;
-    });
-}
-
 function runInBatch(mockList) {
     var running = 0;
 

From b99672b9af6d156aa237412d7ef92cefedb66b0c Mon Sep 17 00:00:00 2001
From: Antoine Roy-Gobeil <antoine@plot.ly>
Date: Mon, 25 Mar 2019 20:46:51 -0400
Subject: [PATCH 4/5] circleci: log which jasmine tests are skipped

---
 .circleci/test.sh          | 6 +++---
 test/jasmine/karma.conf.js | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.circleci/test.sh b/.circleci/test.sh
index 95eee371a92..8094323e6d5 100755
--- a/.circleci/test.sh
+++ b/.circleci/test.sh
@@ -45,7 +45,7 @@ case $1 in
         set_tz
 
         SUITE=$(circleci tests glob "$ROOT/test/jasmine/tests/*" | circleci tests split)
-        npm run test-jasmine -- $SUITE --skip-tags=gl,noCI,flaky || EXIT_STATE=$?
+        npm run test-jasmine -- $SUITE --skip-tags=gl,noCI,flaky --showSkipped || EXIT_STATE=$?
 
         exit $EXIT_STATE
         ;;
@@ -56,7 +56,7 @@ case $1 in
         SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl | circleci tests split))
 
         for s in ${SHARDS[@]}; do
-            retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI
+            retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI --showSkipped
         done
 
         exit $EXIT_STATE
@@ -68,7 +68,7 @@ case $1 in
         SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=flaky | circleci tests split))
 
         for s in ${SHARDS[@]}; do
-            retry npm run test-jasmine -- "$s" --tags=flaky --skip-tags=noCI
+            retry npm run test-jasmine -- "$s" --tags=flaky --skip-tags=noCI --showSkipped
         done
 
         exit $EXIT_STATE
diff --git a/test/jasmine/karma.conf.js b/test/jasmine/karma.conf.js
index 77bc53c0cea..818fd2e1f2b 100644
--- a/test/jasmine/karma.conf.js
+++ b/test/jasmine/karma.conf.js
@@ -59,6 +59,7 @@ if(argv.info) {
         '  - `--nowatch (dflt: `false`, `true` on CI)`: run karma w/o `autoWatch` / multiple run mode',
         '  - `--failFast` (dflt: `false`): exit karma upon first test failure',
         '  - `--verbose` (dflt: `false`): show test result using verbose reporter',
+        '  - `--showSkipped` (dflt: `false`): show tests that are skipped',
         '  - `--tags`: run only test with given tags (using the `jasmine-spec-tags` framework)',
         '  - `--width`(dflt: 1035): set width of the browser window',
         '  - `--height` (dflt: 617): set height of the browser window',
@@ -111,7 +112,7 @@ var pathToCustomMatchers = path.join(__dirname, 'assets', 'custom_matchers.js');
 var pathToUnpolyfill = path.join(__dirname, 'assets', 'unpolyfill.js');
 var pathToMathJax = path.join(constants.pathToDist, 'extras', 'mathjax');
 
-var reporters = (isFullSuite && !argv.tags) ? ['dots', 'spec'] : ['progress'];
+var reporters = ((isFullSuite && !argv.tags) || argv.showSkipped) ? ['dots', 'spec'] : ['progress'];
 if(argv.failFast) reporters.push('fail-fast');
 if(argv.verbose) reporters.push('verbose');
 

From be336489a93f6600c6ac668f4f7b46b323d99e53 Mon Sep 17 00:00:00 2001
From: Antoine Roy-Gobeil <antoine@plot.ly>
Date: Tue, 26 Mar 2019 12:01:48 -0400
Subject: [PATCH 5/5] circleci: :zap: test-image-gl2d, reintroduce sorting of
 gl2d mocks

---
 package.json                      |  2 +-
 test/image/compare_pixels_test.js | 43 ++++++++++++++++++++++++-------
 2 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/package.json b/package.json
index 7bf057fd941..32ba40583b0 100644
--- a/package.json
+++ b/package.json
@@ -38,7 +38,7 @@
     "test-export": "node tasks/test_export.js",
     "test-syntax": "node tasks/test_syntax.js && npm run find-strings -- --no-output",
     "test-bundle": "node tasks/test_bundle.js",
-    "test": "npm run test-jasmine && npm run test-bundle && npm run test-image && npm run test-image-gl2d && npm run test-syntax && npm run lint",
+    "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
     "start-test_dashboard": "node devtools/test_dashboard/server.js",
     "start-image_viewer": "node devtools/image_viewer/server.js",
     "start": "npm run start-test_dashboard",
diff --git a/test/image/compare_pixels_test.js b/test/image/compare_pixels_test.js
index 721af9e8f86..8ed7fa7ff03 100644
--- a/test/image/compare_pixels_test.js
+++ b/test/image/compare_pixels_test.js
@@ -73,12 +73,10 @@ argv._.forEach(function(pattern) {
 });
 
 // To get rid of duplicates
-Array.prototype.unique = function() {
-    return this.filter(function(value, index, self) {
-        return self.indexOf(value) === index;
-    });
-};
-allMockList = allMockList.unique();
+function unique(value, index, self) {
+    return self.indexOf(value) === index;
+}
+allMockList = allMockList.filter(unique);
 
 // filter out untestable mocks if no pattern is specified (ie. we're testing all mocks)
 // or if flag '--filter' is provided
@@ -88,6 +86,8 @@ if(allMock || filter) {
     console.log('\n');
 }
 
+sortGl2dMockList(allMockList);
+
 // main
 if(isInQueue) {
     runInQueue(allMockList);
@@ -99,9 +99,7 @@ else {
 /* Test cases:
  *
  * - font-wishlist
- * - all gl2d
  * - all mapbox
- * - gl3d_cone-*
  *
  * don't behave consistently from run-to-run and/or
  * machine-to-machine; skip over them for now.
@@ -110,7 +108,6 @@ else {
 function untestableFilter(mockName) {
     var cond = !(
         mockName === 'font-wishlist' ||
-        // mockName.indexOf('gl2d_') !== -1 ||
         mockName.indexOf('mapbox_') !== -1
     );
 
@@ -119,6 +116,34 @@ function untestableFilter(mockName) {
     return cond;
 }
 
+/* gl2d pointcloud and other non-regl gl2d mock(s)
+ * must be tested first on in order to work;
+ * sort them here.
+ *
+ * gl-shader appears to conflict with regl.
+ * We suspect that the lone gl context on CircleCI is
+ * having issues with dealing with the two different
+ * program binding algorithm.
+ *
+ * The problem will be solved by switching all our
+ * WebGL-based trace types to regl.
+ *
+ * More info here:
+ * https://github.com/plotly/plotly.js/pull/1037
+ */
+function sortGl2dMockList(mockList) {
+    var mockNames = ['gl2d_pointcloud-basic', 'gl2d_heatmapgl'];
+    var pos = 0;
+
+    mockNames.forEach(function(m) {
+        var ind = mockList.indexOf(m);
+        var tmp = mockList[pos];
+        mockList[pos] = m;
+        mockList[ind] = tmp;
+        pos++;
+    });
+}
+
 function runInBatch(mockList) {
     var running = 0;