From 59ae96ee9c7788d3cf8a02669fe572616c8d6c2b Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Thu, 15 Feb 2018 12:10:40 +0100 Subject: [PATCH 1/2] build: run all unit tests on browserstack * No longer runs tests on Saucelabs, because those jobs mostly time out, or are just flaky. --- .travis.yml | 10 +- package.json | 1 - scripts/browserstack/start-tunnel.sh | 13 +- scripts/ci/sources/mode.sh | 2 +- scripts/ci/sources/tunnel.sh | 18 +-- scripts/ci/travis-testing.sh | 6 +- test/browser-providers.js | 86 +++++------- test/karma.conf.js | 33 +++-- test/remote_browsers.json | 202 ++------------------------- 9 files changed, 86 insertions(+), 285 deletions(-) diff --git a/.travis.yml b/.travis.yml index dbca47fd40e9..e4b8418f5fad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,9 +27,9 @@ jobs: - env: "MODE=payload" - env: "MODE=prerender" - env: "MODE=e2e" - - env: "MODE=saucelabs_required" - - env: "MODE=browserstack_required" - - env: "MODE=travis_required" + - env: "MODE=test-browserstack-1" + - env: "MODE=test-browserstack-2" + - env: "MODE=test-travis-1" - env: "DEPLOY_MODE=build-artifacts" if: type = push - env: "DEPLOY_MODE=docs-content" @@ -45,8 +45,8 @@ env: global: - LOGS_DIR=/tmp/angular-material2-build/logs - SAUCE_USERNAME=angular-ci - - BROWSER_STACK_USERNAME=angularteam1 - - BROWSER_STACK_ACCESS_KEY=BWCd4SynLzdDcv8xtzsB + - BROWSER_STACK_USERNAME=paulgschwendtner2 + - BROWSER_STACK_ACCESS_KEY=hsz2pirQRsDASY1aZKaz - BROWSER_PROVIDER_READY_FILE=/tmp/angular-material2-build/readyfile - BROWSER_PROVIDER_ERROR_FILE=/tmp/angular-material2-build/errorfile diff --git a/package.json b/package.json index 28e4d871e231..f513f9118521 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,6 @@ "karma-coverage": "^1.1.1", "karma-firefox-launcher": "^1.0.1", "karma-jasmine": "^1.1.0", - "karma-sauce-launcher": "^1.2.0", "karma-sourcemap-loader": "^0.3.7", "madge": "^2.2.0", "magic-string": "^0.22.4", diff --git a/scripts/browserstack/start-tunnel.sh b/scripts/browserstack/start-tunnel.sh index 65df8e8949d5..be8260c246a9 100755 --- a/scripts/browserstack/start-tunnel.sh +++ b/scripts/browserstack/start-tunnel.sh @@ -30,14 +30,15 @@ unzip -q ${TUNNEL_FILE} -d browserstack-tunnel # Cleanup the download directory. rm ${TUNNEL_FILE} -ARGS="" +if [ -z "${TRAVIS_JOB_ID}" ]; then + echo "Error: Cannot set up a BrowserStack tunnel if there is no '\$TRAVIS_JOB_ID' set." + exit 1 +fi; -# Set tunnel-id only on Travis, to make local testing easier. -if [ ! -z "${TRAVIS_JOB_ID}" ]; then - ARGS="${ARGS} --local-identifier ${TRAVIS_JOB_ID}" -fi +ARGS="--local-identifier ${TRAVIS_JOB_ID}" -echo "Starting Browserstack Local in the background, logging into: ${TUNNEL_LOG}" +echo "Starting Browserstack Local in the background, logging into: ${TUNNEL_LOG}." \ + "Using the following tunnel identifier: ${TRAVIS_JOB_ID}" # Extension to the BrowserStackLocal binaries, because those can't create a readyfile. function create_ready_file { diff --git a/scripts/ci/sources/mode.sh b/scripts/ci/sources/mode.sh index 6b40a7f5e68b..4940e5885fe3 100644 --- a/scripts/ci/sources/mode.sh +++ b/scripts/ci/sources/mode.sh @@ -22,7 +22,7 @@ is_payload() { } is_unit() { - [[ "${MODE}" =~ ^.*_(optional|required)$ ]] + [[ "${MODE}" =~ ^test-.* ]] } is_prerender() { diff --git a/scripts/ci/sources/tunnel.sh b/scripts/ci/sources/tunnel.sh index 5256422d5073..bc9a60c6dac5 100644 --- a/scripts/ci/sources/tunnel.sh +++ b/scripts/ci/sources/tunnel.sh @@ -6,12 +6,12 @@ source ./scripts/retry-call.sh # Variable the specifies how often the wait script should be invoked if it fails. WAIT_RETRIES=2 -start_tunnel() { +start_tunnel_if_necessary() { case "$MODE" in - e2e*|saucelabs*) + e2e*) ./scripts/saucelabs/start-tunnel.sh ;; - browserstack*) + test-browserstack-*) ./scripts/browserstack/start-tunnel.sh ;; *) @@ -19,12 +19,12 @@ start_tunnel() { esac } -wait_for_tunnel() { +wait_for_tunnel_if_present() { case "$MODE" in - e2e*|saucelabs*) + e2e*) retryCall ${WAIT_RETRIES} ./scripts/saucelabs/wait-tunnel.sh ;; - browserstack*) + test-browserstack-*) retryCall ${WAIT_RETRIES} ./scripts/browserstack/wait-tunnel.sh ;; *) @@ -32,12 +32,12 @@ wait_for_tunnel() { esac } -teardown_tunnel() { +teardown_tunnel_if_present() { case "$MODE" in - e2e*|saucelabs*) + e2e*) ./scripts/saucelabs/stop-tunnel.sh ;; - browserstack*) + test-browserstack-*) ./scripts/browserstack/stop-tunnel.sh ;; *) diff --git a/scripts/ci/travis-testing.sh b/scripts/ci/travis-testing.sh index 9ab10831ba99..14104abadc0c 100755 --- a/scripts/ci/travis-testing.sh +++ b/scripts/ci/travis-testing.sh @@ -28,8 +28,8 @@ if [ "$TRAVIS_PULL_REQUEST" = "true" ]; then fi fi -start_tunnel -wait_for_tunnel +start_tunnel_if_necessary +wait_for_tunnel_if_present if is_lint; then $(npm bin)/gulp ci:lint @@ -52,4 +52,4 @@ if [ -f dist/coverage/coverage-summary.json ]; then $(npm bin)/gulp ci:coverage fi -teardown_tunnel +teardown_tunnel_if_present diff --git a/test/browser-providers.js b/test/browser-providers.js index d475817c573f..46fcaa081e6f 100644 --- a/test/browser-providers.js +++ b/test/browser-providers.js @@ -5,30 +5,17 @@ * Target can be either: BS (Browserstack) | SL (Saucelabs) | TC (Travis CI) | null (To not run) */ const browserConfig = { - 'ChromeHeadlessCI': { unitTest: {target: 'TC', required: true }}, - 'FirefoxHeadless': { unitTest: {target: 'TC', required: true }}, - 'ChromeBeta': { unitTest: {target: null, required: false }}, - 'FirefoxBeta': { unitTest: {target: null, required: false }}, - 'ChromeDev': { unitTest: {target: null, required: true }}, - 'FirefoxDev': { unitTest: {target: null, required: true }}, - 'IE9': { unitTest: {target: null, required: false }}, - 'IE10': { unitTest: {target: null, required: true }}, - 'IE11': { unitTest: {target: 'SL', required: true }}, - 'Edge': { unitTest: {target: 'SL', required: true }}, - 'Android4.1': { unitTest: {target: null, required: false }}, - 'Android4.2': { unitTest: {target: null, required: false }}, - 'Android4.3': { unitTest: {target: null, required: false }}, - 'Android4.4': { unitTest: {target: null, required: false }}, - 'Android5': { unitTest: {target: null, required: false }}, - 'Safari7': { unitTest: {target: null, required: false }}, - 'Safari8': { unitTest: {target: null, required: false }}, - 'Safari9': { unitTest: {target: 'SL', required: true }}, - 'Safari10': { unitTest: {target: 'BS', required: true }}, - 'iOS7': { unitTest: {target: null, required: false }}, - 'iOS8': { unitTest: {target: null, required: false }}, - 'iOS9': { unitTest: {target: null, required: false }}, - 'iOS10': { unitTest: {target: 'BS', required: true }}, - 'WindowsPhone': { unitTest: {target: null, required: false }} + 'ChromeHeadlessCI': { target: 'TC', poolId: 1 }, + 'FirefoxHeadless': { target: 'TC', poolId: 1 }, + 'IE11': { target: null }, + 'Edge': { target: 'BS', poolId: 1 }, + 'Android4.4': { target: null }, + 'Android5': { target: null }, + 'Safari10': { target: 'BS', poolId: 1 }, + 'Safari11': { target: 'BS', poolId: 2 }, + 'iOS10': { target: null }, + 'iOS11': { target: 'BS', poolId: 2 }, + 'WindowsPhone': { target: null } }; /** Exports all available remote browsers. */ @@ -36,34 +23,35 @@ exports.customLaunchers = require('./remote_browsers.json'); /** Exports a map of configured browsers, which should run on the CI. */ exports.platformMap = { - 'saucelabs': { - required: buildConfiguration('unitTest', 'SL', true), - optional: buildConfiguration('unitTest', 'SL', false) - }, - 'browserstack': { - required: buildConfiguration('unitTest', 'BS', true), - optional: buildConfiguration('unitTest', 'BS', false) - }, - 'travis': { - required: buildConfiguration('unitTest', 'TC', true), - optional: buildConfiguration('unitTest', 'TC', false) - } + 'browserstack': buildConfiguration('BS'), + 'travis': buildConfiguration('TC'), }; -/** Build a list of configuration (custom launcher names). */ -function buildConfiguration(type, target, required) { - const targetBrowsers = Object.keys(browserConfig) - .map(browserName => [browserName, browserConfig[browserName][type]]) - .filter(([, config]) => config.required === required && config.target === target) - .map(([browserName]) => browserName); +/** Build a list of configuration for the specified platform. */ +function buildConfiguration(platform) { + const platformConfig = {}; - // For browsers that run on Travis CI the browser name shouldn't be prefixed with the shortcut - // of Travis. The different Karma launchers only work with the plain browser name (e.g Firefox) - if (target === 'TC') { - return targetBrowsers; - } + Object.keys(browserConfig).forEach(browserName => { + const config = browserConfig[browserName]; - return targetBrowsers.map(browserName => `${target}_${browserName.toUpperCase()}`); + if (config.target !== platform || !config.poolId) { + return; + } + + if (!platformConfig[config.poolId]) { + platformConfig[config.poolId] = []; + } + + // For browsers that run on Travis CI the browser name shouldn't be prefixed with the shortcut + // of Travis. The different Karma launchers only work with the plain browser name (e.g Firefox) + if (platform !== 'TC') { + browserName = `${platform}_${browserName.toUpperCase()}`; + } + + platformConfig[config.poolId].push(browserName); + }); + + return platformConfig; } /** Decode the token for Travis to use. */ @@ -71,9 +59,7 @@ function decodeToken(token) { return (token || '').split('').reverse().join(''); } - /** Ensures that the Travis access keys work properly. */ if (process.env.TRAVIS) { - process.env.SAUCE_ACCESS_KEY = decodeToken(process.env.SAUCE_ACCESS_KEY); process.env.BROWSER_STACK_ACCESS_KEY = decodeToken(process.env.BROWSER_STACK_ACCESS_KEY); } diff --git a/test/karma.conf.js b/test/karma.conf.js index c1440ad28ec4..75a6d276411d 100644 --- a/test/karma.conf.js +++ b/test/karma.conf.js @@ -9,7 +9,6 @@ module.exports = (config) => { plugins: [ require('karma-jasmine'), require('karma-browserstack-launcher'), - require('karma-sauce-launcher'), require('karma-chrome-launcher'), require('karma-firefox-launcher'), require('karma-sourcemap-loader'), @@ -79,9 +78,10 @@ module.exports = (config) => { browserDisconnectTimeout: 20000, browserNoActivityTimeout: 240000, + browserDisconnectTolerance: 1, captureTimeout: 120000, - browsers: ['ChromeHeadlessLocal'], + browsers: ['ChromeHeadlessLocal'], singleRun: false, browserConsoleLogOptions: { @@ -98,31 +98,30 @@ module.exports = (config) => { }); if (process.env['TRAVIS']) { - const buildId = `TRAVIS #${process.env.TRAVIS_BUILD_NUMBER} (${process.env.TRAVIS_BUILD_ID})`; + const buildId = `TravisCI ${process.env.TRAVIS_BUILD_ID}`; + const tunnelIdentifier = process.env.TRAVIS_JOB_ID; - if (process.env['TRAVIS_PULL_REQUEST'] === 'false' && - process.env['MODE'] === "travis_required") { + if (process.env['TRAVIS_PULL_REQUEST'] === 'false' + && process.env['MODE'] === "test-travis-1") { config.preprocessors['dist/packages/**/!(*+(.|-)spec).js'] = ['coverage']; config.reporters.push('coverage'); } // The MODE variable is the indicator of what row in the test matrix we're running. - // It will look like _, where platform is one of 'saucelabs', 'browserstack' - // or 'travis'. The target is a reference to different collections of browsers that can run - // in the previously specified platform. - const [platform, target] = process.env.MODE.split('_'); - - if (platform === 'saucelabs') { - config.sauceLabs.build = buildId; - config.sauceLabs.tunnelIdentifier = process.env.TRAVIS_JOB_ID; - } else if (platform === 'browserstack') { + // It will look like "test-[-poolId], where platform is one of 'browserstack' or + // 'travis'. Pool ids allow running different browsers on the same + // platform, but in different CI jobs. + const [, platform, poolId] = process.env.MODE.split('-'); + + if (platform === 'browserstack') { config.browserStack.build = buildId; - config.browserStack.tunnelIdentifier = process.env.TRAVIS_JOB_ID; + config.browserStack.tunnelIdentifier = tunnelIdentifier; + console.log(`Setting up Browserstack launcher. Connecting to tunnel: "${tunnelIdentifier}"`); } else if (platform !== 'travis') { - throw new Error(`Platform "${platform}" unknown, but Travis specified. Exiting.`); + throw new Error(`Platform "${platform}" is unknown. Exiting..`); } - config.browsers = platformMap[platform][target.toLowerCase()]; + config.browsers = platformMap[platform][parseInt(poolId)]; } }; diff --git a/test/remote_browsers.json b/test/remote_browsers.json index e0852b1aec93..0c9a870a11f0 100644 --- a/test/remote_browsers.json +++ b/test/remote_browsers.json @@ -18,132 +18,6 @@ "-headless" ] }, - "SL_CHROME": { - "base": "SauceLabs", - "browserName": "chrome", - "version": "latest" - }, - "SL_CHROMEBETA": { - "base": "SauceLabs", - "browserName": "chrome", - "version": "beta" - }, - "SL_CHROMEDEV": { - "base": "SauceLabs", - "browserName": "chrome", - "version": "dev" - }, - "SL_FIREFOX": { - "base": "SauceLabs", - "browserName": "firefox", - "version": "latest" - }, - "SL_FIREFOXBETA": { - "base": "SauceLabs", - "browserName": "firefox", - "version": "beta" - }, - "SL_FIREFOXDEV": { - "base": "SauceLabs", - "browserName": "firefox", - "version": "dev" - }, - "SL_SAFARI7": { - "base": "SauceLabs", - "browserName": "safari", - "platform": "OS X 10.9", - "version": "7" - }, - "SL_SAFARI8": { - "base": "SauceLabs", - "browserName": "safari", - "platform": "OS X 10.10", - "version": "8" - }, - "SL_SAFARI9": { - "base": "SauceLabs", - "browserName": "safari", - "platform": "OS X 10.11", - "version": "9.0" - }, - "SL_SAFARI10": { - "base": "SauceLabs", - "browserName": "safari", - "platform": "OS X 10.12", - "version": "10.0" - }, - "SL_IOS7": { - "base": "SauceLabs", - "browserName": "iphone", - "platform": "OS X 10.10", - "version": "7.1" - }, - "SL_IOS8": { - "base": "SauceLabs", - "browserName": "iphone", - "platform": "OS X 10.10", - "version": "8.4" - }, - "SL_IOS9": { - "base": "SauceLabs", - "browserName": "iphone", - "platform": "OS X 10.10", - "version": "9.1" - }, - "SL_IE9": { - "base": "SauceLabs", - "browserName": "internet explorer", - "platform": "Windows 2008", - "version": "9" - }, - "SL_IE10": { - "base": "SauceLabs", - "browserName": "internet explorer", - "platform": "Windows 2012", - "version": "10" - }, - "SL_IE11": { - "base": "SauceLabs", - "browserName": "internet explorer", - "platform": "Windows 8.1", - "version": "11" - }, - "SL_EDGE": { - "base": "SauceLabs", - "browserName": "microsoftedge", - "platform": "Windows 10", - "version": "14" - }, - "SL_ANDROID4.1": { - "base": "SauceLabs", - "browserName": "android", - "platform": "Linux", - "version": "4.1" - }, - "SL_ANDROID4.2": { - "base": "SauceLabs", - "browserName": "android", - "platform": "Linux", - "version": "4.2" - }, - "SL_ANDROID4.3": { - "base": "SauceLabs", - "browserName": "android", - "platform": "Linux", - "version": "4.3" - }, - "SL_ANDROID4.4": { - "base": "SauceLabs", - "browserName": "android", - "platform": "Linux", - "version": "4.4" - }, - "SL_ANDROID5": { - "base": "SauceLabs", - "browserName": "android", - "platform": "Linux", - "version": "5.1" - }, "BS_CHROME": { "base": "BrowserStack", "browser": "chrome", @@ -156,24 +30,6 @@ "os": "Windows", "os_version": "10" }, - "BS_SAFARI7": { - "base": "BrowserStack", - "browser": "safari", - "os": "OS X", - "os_version": "Mavericks" - }, - "BS_SAFARI8": { - "base": "BrowserStack", - "browser": "safari", - "os": "OS X", - "os_version": "Yosemite" - }, - "BS_SAFARI9": { - "base": "BrowserStack", - "browser": "safari", - "os": "OS X", - "os_version": "El Capitan" - }, "BS_SAFARI10": { "base": "BrowserStack", "browser": "safari", @@ -181,26 +37,12 @@ "os": "OS X", "os_version": "Sierra" }, - "BS_IOS7": { + "BS_SAFARI11": { "base": "BrowserStack", - "device": "iPhone 5S", - "os": "ios", - "os_version": "7.0", - "resolution": "1024x768" - }, - "BS_IOS8": { - "base": "BrowserStack", - "device": "iPhone 6", - "os": "ios", - "os_version": "8.3", - "resolution": "1024x768" - }, - "BS_IOS9": { - "base": "BrowserStack", - "device": "iPhone 6S", - "os": "ios", - "os_version": "9.0", - "resolution": "1024x768" + "browser": "safari", + "browser_version": "11.0", + "os": "OS X", + "os_version": "High Sierra" }, "BS_IOS10": { "base": "BrowserStack", @@ -208,19 +50,11 @@ "os": "ios", "os_version": "10.0" }, - "BS_IE9": { - "base": "BrowserStack", - "browser": "ie", - "browser_version": "9.0", - "os": "Windows", - "os_version": "7" - }, - "BS_IE10": { + "BS_IOS11": { "base": "BrowserStack", - "browser": "ie", - "browser_version": "10.0", - "os": "Windows", - "os_version": "8" + "device": "iPhone 8", + "os": "ios", + "os_version": "11.0" }, "BS_IE11": { "base": "BrowserStack", @@ -252,23 +86,5 @@ "device": "HTC One M8", "os": "android", "os_version": "4.4" - }, - "BS_ANDROID4.3": { - "base": "BrowserStack", - "device": "Samsung Galaxy S4", - "os": "android", - "os_version": "4.3" - }, - "BS_ANDROID4.2": { - "base": "BrowserStack", - "device": "Google Nexus 4", - "os": "android", - "os_version": "4.2" - }, - "BS_ANDROID4.1": { - "base": "BrowserStack", - "device": "Google Nexus 7", - "os": "android", - "os_version": "4.1" } } From 3f7d180554bd05cacccd936f1a7ad28c98de6f47 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Sat, 17 Feb 2018 16:05:28 +0100 Subject: [PATCH 2/2] Change browserstack login --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e4b8418f5fad..628d3878e909 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,8 +45,8 @@ env: global: - LOGS_DIR=/tmp/angular-material2-build/logs - SAUCE_USERNAME=angular-ci - - BROWSER_STACK_USERNAME=paulgschwendtner2 - - BROWSER_STACK_ACCESS_KEY=hsz2pirQRsDASY1aZKaz + - BROWSER_STACK_USERNAME=angularteam1 + - BROWSER_STACK_ACCESS_KEY=BWCd4SynLzdDcv8xtzsB - BROWSER_PROVIDER_READY_FILE=/tmp/angular-material2-build/readyfile - BROWSER_PROVIDER_ERROR_FILE=/tmp/angular-material2-build/errorfile