Flaky test-runner-run-watch

[tniessen]

Test

test-runner-run-watch

Platform

Windows

Console output

16:41:02 ok 848 parallel/test-vfs
16:41:02   ---
16:44:04   duration_ms: 1255.99700
16:44:04   ...
16:44:04  failed 10 out of 10
16:44:04 not ok 849 parallel/test-runner-run-watch
16:44:04   ---
16:44:04   duration_ms: 121964.85900
16:44:04   severity: fail
16:44:04   exitcode: 1
16:44:04   stack: |-
16:44:04     timeout
16:44:04     TAP version 13
16:44:04     # Subtest: test runner watch mode
16:44:04         # Subtest: should run tests repeatedly
16:44:04         not ok 1 - should run tests repeatedly
16:44:04           ---
16:44:04           duration_ms: 11682.5747
16:44:04           location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\parallel\\test-runner-run-watch.mjs:155:3'
16:44:04           failureType: 'testCodeFailure'
16:44:04           error: |-
16:44:04             The input did not match the regular expression /# tests 1/. Input:
16:44:04             
16:44:04             '# Subtest: test has ran\n' +
16:44:04               'ok 1 - test has ran\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 37.95\n' +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 2 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 1068.7518\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 3 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 1050.696\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 4 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 1012.4334\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 5 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 974.3208\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 6 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 999.9954\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 7 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 1011.2067\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test has ran\n' +
16:44:04               'ok 8 - test has ran\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 29.1914\n' +
16:44:04               '  ...\n' +
16:44:04               '1..8\n' +
16:44:04               '# tests 8\n' +
16:44:04               '# suites 0\n' +
16:44:04               '# pass 2\n' +
16:44:04               '# fail 6\n' +
16:44:04               '# cancelled 0\n' +
16:44:04               '# skipped 0\n' +
16:44:04               '# todo 0\n' +
16:44:04               '# duration_ms 1693.5803\n'
16:44:04             
16:44:04           code: 'ERR_ASSERTION'
16:44:04           name: 'AssertionError'
16:44:04           expected:
16:44:04           actual: |-
16:44:04             # Subtest: test has ran
16:44:04             ok 1 - test has ran
16:44:04               ---
16:44:04               duration_ms: 37.95
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 2 - test.js
16:44:04               ---
16:44:04               duration_ms: 1068.7518
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 3 - test.js
16:44:04               ---
16:44:04               duration_ms: 1050.696
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 4 - test.js
16:44:04               ---
16:44:04               duration_ms: 1012.4334
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 5 - test.js
16:44:04               ---
16:44:04               duration_ms: 974.3208
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 6 - test.js
16:44:04               ---
16:44:04               duration_ms: 999.9954
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 7 - test.js
16:44:04               ---
16:44:04               duration_ms: 1011.2067
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test has ran
16:44:04             ok 8 - test has ran
16:44:04               ---
16:44:04               duration_ms: 29.1914
16:44:04               ...
16:44:04             1..8
16:44:04             # tests 8
16:44:04             # suites 0
16:44:04             # pass 2
16:44:04             # fail 6
16:44:04             # cancelled 0
16:44:04             # skipped 0
16:44:04             # todo 0
16:44:04             # duration_ms 1693.5803
16:44:04             
16:44:04           operator: 'match'
16:44:04           stack: |-
16:44:04             testUpdate (file:///c:/workspace/node-test-binary-windows-js-suites/node/test/parallel/test-runner-run-watch.mjs:81:14)
16:44:04             process.processTicksAndRejections (node:internal/process/task_queues:105:5)
16:44:04             async testWatch (file:///c:/workspace/node-test-binary-windows-js-suites/node/test/parallel/test-runner-run-watch.mjs:147:26)
16:44:04             async TestContext.<anonymous> (file:///c:/workspace/node-test-binary-windows-js-suites/node/test/parallel/test-runner-run-watch.mjs:156:5)
16:44:04             async Test.run (node:internal/test_runner/test:879:9)
16:44:04             async Promise.all (index 0)
16:44:04             async Suite.run (node:internal/test_runner/test:1239:7)
16:44:04             async Test.processPendingSubtests (node:internal/test_runner/test:590:7)
16:44:04           ...
16:44:04         # Subtest: should run tests with dependency repeatedly
16:44:04         not ok 2 - should run tests with dependency repeatedly
16:44:04           ---
16:44:04           duration_ms: 9670.1542
16:44:04           location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\parallel\\test-runner-run-watch.mjs:159:3'
16:44:04           failureType: 'testCodeFailure'
16:44:04           error: |-
16:44:04             The input did not match the regular expression /# tests 1/. Input:
16:44:04             
16:44:04             '# Subtest: test has ran\n' +
16:44:04               'ok 1 - test has ran\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 10.4875\n' +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 2 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 1085.3394\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 3 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 1111.3339\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 4 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 954.7013\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 5 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 1070.1657\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test.js\n' +
16:44:04               'not ok 6 - test.js\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 926.3438\n' +
16:44:04               "  location: 'c:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.682\\\\test.js:1:1'\n" +
16:44:04               "  failureType: 'testCodeFailure'\n" +
16:44:04               '  exitCode: ~\n' +
16:44:04               "  signal: 'SIGTERM'\n" +
16:44:04               "  error: 'test failed'\n" +
16:44:04               "  code: 'ERR_TEST_FAILURE'\n" +
16:44:04               '  ...\n' +
16:44:04               '# Subtest: test has ran\n' +
16:44:04               'ok 7 - test has ran\n' +
16:44:04               '  ---\n' +
16:44:04               '  duration_ms: 26.637\n' +
16:44:04               '  ...\n' +
16:44:04               '1..7\n' +
16:44:04               '# tests 7\n' +
16:44:04               '# suites 0\n' +
16:44:04               '# pass 2\n' +
16:44:04               '# fail 5\n' +
16:44:04               '# cancelled 0\n' +
16:44:04               '# skipped 0\n' +
16:44:04               '# todo 0\n' +
16:44:04               '# duration_ms 1325.8181\n'
16:44:04             
16:44:04           code: 'ERR_ASSERTION'
16:44:04           name: 'AssertionError'
16:44:04           expected:
16:44:04           actual: |-
16:44:04             # Subtest: test has ran
16:44:04             ok 1 - test has ran
16:44:04               ---
16:44:04               duration_ms: 10.4875
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 2 - test.js
16:44:04               ---
16:44:04               duration_ms: 1085.3394
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 3 - test.js
16:44:04               ---
16:44:04               duration_ms: 1111.3339
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 4 - test.js
16:44:04               ---
16:44:04               duration_ms: 954.7013
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 5 - test.js
16:44:04               ---
16:44:04               duration_ms: 1070.1657
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test.js
16:44:04             not ok 6 - test.js
16:44:04               ---
16:44:04               duration_ms: 926.3438
16:44:04               location: 'c:\\workspace\\node-test-binary-windows-js-suites\\node\\test\\.tmp.682\\test.js:1:1'
16:44:04               failureType: 'testCodeFailure'
16:44:04               exitCode: ~
16:44:04               signal: 'SIGTERM'
16:44:04               error: 'test failed'
16:44:04               code: 'ERR_TEST_FAILURE'
16:44:04               ...
16:44:04             # Subtest: test has ran
16:44:04             ok 7 - test has ran
16:44:04               ---
16:44:04               duration_ms: 26.637
16:44:04               ...
16:44:04             1..7
16:44:04             # tests 7
16:44:04             # suites 0
16:44:04             # pass 2
16:44:04             # fail 5
16:44:04             # cancelled 0
16:44:04             # skipped 0
16:44:04             # todo 0
16:44:04             # duration_ms 1325.8181
16:44:04             
16:44:04           operator: 'match'
16:44:04           stack: |-
16:44:04             testUpdate (file:///c:/workspace/node-test-binary-windows-js-suites/node/test/parallel/test-runner-run-watch.mjs:81:14)
16:44:04             process.processTicksAndRejections (node:internal/process/task_queues:105:5)
16:44:04             async testWatch (file:///c:/workspace/node-test-binary-windows-js-suites/node/test/parallel/test-runner-run-watch.mjs:147:26)
16:44:04             async TestContext.<anonymous> (file:///c:/workspace/node-test-binary-windows-js-suites/node/test/parallel/test-runner-run-watch.mjs:160:5)
16:44:04             async Test.run (node:internal/test_runner/test:879:9)
16:44:04             async Suite.processPendingSubtests (node:internal/test_runner/test:590:7)
16:44:04           ...
16:44:04         # Subtest: should run tests with ESM dependency
16:44:04         ok 3 - should run tests with ESM dependency
16:44:04           ---
16:44:04           duration_ms: 5620.9341
16:44:04           ...
16:44:04   ...

Build links

• https://ci.nodejs.org/job/node-test-binary-windows-js-suites/29397/RUN_SUBSET=2,nodes=win10-COMPILED_BY-vs2022/

Additional information

This might be related to #44898 and #49605.

[anonrig]

It is also flaky on https://ci.nodejs.org/job/node-test-binary-windows-js-suites/30010/RUN_SUBSET=2,nodes=win10-COMPILED_BY-vs2022/testReport/junit/(root)/parallel/test_runner_run_watch/

[anonrig]

It is also flaky on smartOS: https://ci.nodejs.org/job/node-test-commit-smartos/56585/

[mhdawson]

Failed on macOS as well. https://github.com/nodejs/node/actions/runs/13041166114/job/36383144987?pr=56819

[tniessen]

Yet another failure on Windows:

---
duration_ms: 28588.968
exitcode: 1
severity: flaky
stack: |-
  Test failure: 'should run tests repeatedly'
  Location: test\parallel\test-runner-run-watch.mjs:198:3
  AssertionError [ERR_ASSERTION]: The input did not match the regular expression /tests 1/. Input:

  '# Subtest: test.js\n' +
    'not ok 2 - test.js\n' +
    '  ---\n' +
    '  duration_ms: 1097.7679\n' +
    "  type: 'test'\n" +
    "  location: 'C:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.726\\\\test.js:1:1'\n" +
    "  failureType: 'testCodeFailure'\n" +
    '  exitCode: ~\n' +
    "  signal: 'SIGTERM'\n" +
    "  error: 'test failed'\n" +
    "  code: 'ERR_TEST_FAILURE'\n" +
    '  ...\n' +
    '# Subtest: test has ran\n' +
    'ok 2 - test has ran\n' +
    '  ---\n' +
    '  duration_ms: 2.7913\n' +
    "  type: 'test'\n" +
    '  ...\n' +
    '1..2\n' +
    '# tests 2\n' +
    '# suites 0\n' +
    '# pass 1\n' +
    '# fail 1\n' +
    '# cancelled 0\n' +
    '# skipped 0\n' +
    '# todo 0\n' +
    '# duration_ms 975.6305\n'

      at testUpdate (file:///C:/workspace/node-test-binary-windows-js-suites/node/test/parallel/test-runner-run-watch.mjs:91:14)
      at process.processTicksAndRejections (node:internal/process/task_queues:105:5)
      at async testWatch (file:///C:/workspace/node-test-binary-windows-js-suites/node/test/parallel/test-runner-run-watch.mjs:187:26)
      at async TestContext.<anonymous> (file:///C:/workspace/node-test-binary-windows-js-suites/node/test/parallel/test-runner-run-watch.mjs:199:5)
      at async Test.run (node:internal/test_runner/test:1069:7)
      at async Promise.all (index 0)
      at async Suite.run (node:internal/test_runner/test:1466:7)
      at async startSubtestAfterBootstrap (node:internal/test_runner/harness:332:3) {
    generatedMessage: true,
    code: 'ERR_ASSERTION',
    actual: '# Subtest: test.js\n' +
      'not ok 2 - test.js\n' +
      '  ---\n' +
      '  duration_ms: 1097.7679\n' +
      "  type: 'test'\n" +
      "  location: 'C:\\\\workspace\\\\node-test-binary-windows-js-suites\\\\node\\\\test\\\\.tmp.726\\\\test.js:1:1'\n" +
      "  failureType: 'testCodeFailure'\n" +
      '  exitCode: ~\n' +
      "  signal: 'SIGTERM'\n" +
      "  error: 'test failed'\n" +
      '...',
    expected: /tests 1/,
    operator: 'match'
  }
...

https://ci.nodejs.org/job/node-test-binary-windows-js-suites/34341/RUN_SUBSET=1,nodes=win2019-COMPILED_BY-vs2022_clang/testReport/junit/(root)/parallel/test_runner_run_watch_/