random feature execution order (#1055)

Author: charlierudolph

CHANGELOG.md

@@ -2,6 +2,8 @@ Please see [CONTRIBUTING.md](https://github.com/cucumber/cucumber/blob/master/CO
 
 ### [Unreleased](https://github.com/cucumber/cucumber-js/compare/v4.1.0...master) (In Git)
 
+* add cli option `--order <TYPE[:SEED]>` to run scenarios in the specified order. Type should be `defined` or `random`
+
 ### [4.1.0](https://github.com/cucumber/cucumber-js/compare/v4.0.0...v4.1.0) (2018-03-27)
 
 #### New Features

features/order.feature

@@ -0,0 +1,49 @@
+Feature: Set the execution order
+
+  Background:
+    Given a file named "features/a.feature" with:
+      """
+      Feature: some feature
+        @a
+        Scenario: first scenario
+          Given a step
+
+        @b
+        Scenario Outline: second scenario - <ID>
+          Given a step
+
+          @c
+          Examples:
+            | ID |
+            | X  |
+            | Y  |
+
+          @d
+          Examples:
+            | ID |
+            | Z  |
+      """
+    And a file named "features/step_definitions/cucumber_steps.js" with:
+      """
+      import {Given} from 'cucumber'
+
+      Given(/^a step$/, function() {})
+      """
+
+  Scenario: run in defined order scenario
+    When I run cucumber-js with `--order defined`
+    Then it runs the scenarios:
+      | NAME                |
+      | first scenario      |
+      | second scenario - X |
+      | second scenario - Y |
+      | second scenario - Z |
+
+  Scenario: run in random order with seed
+    When I run cucumber-js with `--order random:234119`
+    Then it runs the scenarios:
+      | NAME                |
+      | second scenario - Z |
+      | second scenario - X |
+      | second scenario - Y |
+      | first scenario      |

package.json

@@ -155,6 +155,7 @@
     "indent-string": "^3.1.0",
     "is-generator": "^1.0.2",
     "is-stream": "^1.1.0",
+    "knuth-shuffle-seeded": "^1.0.6",
     "lodash": "^4.17.4",
     "mz": "^2.4.0",
     "progress": "^2.0.0",

src/cli/argv_parser.js

@@ -82,6 +82,11 @@ export default class ArgvParser {
         []
       )
       .option('--no-strict', 'succeed even if there are pending steps')
+      .option(
+        '--order <TYPE[:SEED]>',
+        'run scenarios in the specified order. Type should be `defined` or `random`',
+        'defined'
+      )
       .option(
         '-p, --profile <NAME>',
         'specify the profile to use (repeatable)',

src/cli/configuration_builder.js

@@ -46,6 +46,7 @@ export default class ConfigurationBuilder {
       formatOptions: this.getFormatOptions(),
       listI18nKeywordsFor,
       listI18nLanguages,
+      order: this.options.order,
       parallel: this.options.parallel,
       profiles: this.options.profile,
       pickleFilterOptions: {

src/cli/configuration_builder_spec.js

@@ -35,6 +35,7 @@ describe('Configuration', () => {
         formats: [{ outputTo: '', type: 'progress' }],
         listI18nKeywordsFor: '',
         listI18nLanguages: false,
+        order: 'defined',
         parallel: 0,
         pickleFilterOptions: {
           featurePaths: ['features/**/*.feature'],

src/cli/helpers.js

@@ -5,6 +5,7 @@ import Gherkin from 'gherkin'
 import path from 'path'
 import ProfileLoader from './profile_loader'
 import Promise from 'bluebird'
+import shuffle from 'knuth-shuffle-seeded'
 
 export async function getExpandedArgv({ argv, cwd }) {
   const { options } = ArgvParser.parse(argv)
@@ -21,6 +22,7 @@ export async function getTestCasesFromFilesystem({
   eventBroadcaster,
   featureDefaultLanguage,
   featurePaths,
+  order,
   pickleFilter,
 }) {
   let result = []
@@ -36,6 +38,7 @@ export async function getTestCasesFromFilesystem({
       })
     )
   })
+  orderTestCases(result, order)
   return result
 }
 
@@ -65,3 +68,23 @@ export async function getTestCases({
   })
   return result
 }
+
+// Orders the testCases in place - morphs input
+export function orderTestCases(testCases, order) {
+  let [type, seed] = order.split(':')
+  switch (type) {
+    case 'defined':
+      break
+    case 'random':
+      if (!seed) {
+        seed = Math.floor(Math.random() * 1000 * 1000).toString()
+        console.warn(`Random order using seed: ${seed}`)
+      }
+      shuffle(testCases, seed)
+      break
+    default:
+      throw new Error(
+        'Unrecgonized order type. Should be `defined` or `random`'
+      )
+  }
+}

src/cli/helpers_spec.js

@@ -35,6 +35,7 @@ describe('helpers', () => {
           cwd: this.tmpDir,
           eventBroadcaster: this.eventBroadcaster,
           featurePaths: [featurePath],
+          order: 'defined',
           pickleFilter: new PickleFilter({}),
         })
       })
@@ -79,6 +80,7 @@ describe('helpers', () => {
           cwd: this.tmpDir,
           eventBroadcaster: this.eventBroadcaster,
           featurePaths: [featurePath],
+          order: 'defined',
           pickleFilter: new PickleFilter({
             featurePaths: [`${this.relativeFeaturePath}:5`],
           }),
@@ -119,6 +121,7 @@ describe('helpers', () => {
           cwd: this.tmpDir,
           eventBroadcaster: this.eventBroadcaster,
           featurePaths: [featurePath],
+          order: 'defined',
           pickleFilter: new PickleFilter({}),
         })
       })

src/cli/index.js

@@ -88,6 +88,7 @@ export default class Cli {
       eventBroadcaster,
       featureDefaultLanguage: configuration.featureDefaultLanguage,
       featurePaths: configuration.featurePaths,
+      order: configuration.order,
       pickleFilter: new PickleFilter(configuration.pickleFilterOptions),
     })
     let success

src/index.js

@@ -3,7 +3,11 @@ import supportCodeLibraryBuilder from './support_code_library_builder'
 
 // Top level
 export { default as Cli } from './cli'
-export { getTestCases, getTestCasesFromFilesystem } from './cli/helpers'
+export {
+  getTestCases,
+  getTestCasesFromFilesystem,
+  orderTestCases,
+} from './cli/helpers'
 export { default as PickleFilter } from './pickle_filter'
 export { default as Runtime } from './runtime'
 export { default as Status } from './status'