diff --git a/docs/05-command-line.md b/docs/05-command-line.md index a92ed2654..1097b3050 100644 --- a/docs/05-command-line.md +++ b/docs/05-command-line.md @@ -37,6 +37,8 @@ Options: --no-worker-threads Don't use worker threads [boolean] --node-arguments Additional Node.js arguments for launching worker processes (specify as a single string) [string] + --randomize Randomize test file and test order [boolean] + --seed Seed for randomized test order [string] -s, --serial Run tests serially [boolean] -t, --tap Generate TAP output [boolean] -T, --timeout Set global timeout (milliseconds or human-readable, @@ -80,6 +82,22 @@ Files inside `node_modules` are *always* ignored. So are files starting with `_` When using `npm test`, you can pass positional arguments directly `npm test test2.js`, but flags needs to be passed like `npm test -- --verbose`. +## Randomizing test order + +Use the `--randomize` flag to run test files, and concurrent tests within each file, in a random order. AVA reports the seed used for each run: + +```console +npx ava --randomize +``` + +Use `--seed` to reproduce the same order: + +```console +npx ava --seed=4f7a2c91 +``` + +Serial tests still run in declaration order. + ## Running tests with matching titles [![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/avajs/ava/tree/main/examples/matching-titles?file=test.js&terminal=test&view=editor) diff --git a/docs/06-configuration.md b/docs/06-configuration.md index c1c769b3c..dccdf3448 100644 --- a/docs/06-configuration.md +++ b/docs/06-configuration.md @@ -51,6 +51,8 @@ Arguments passed to the CLI will always take precedence over the CLI options con - `failFast`: stop running further tests once a test fails - `failWithoutAssertions`: if `false`, does not fail a test if it doesn't run [assertions](./03-assertions.md) - `environmentVariables`: specifies environment variables to be made available to the tests. The environment variables defined here override the ones from `process.env` +- `randomize`: if `true`, randomizes test file order and concurrent test order within each file. Serial tests still run in declaration order +- `seed`: sets the seed used for randomized test order. This implies `randomize`, unless `randomize` is `false` - `serial`: if `true`, prevents parallel execution of tests within a file - `tap`: if `true`, enables the [TAP reporter](./05-command-line.md#tap-reporter) - `verbose`: if `true`, enables verbose output (though there currently non-verbose output is not supported) diff --git a/lib/api.js b/lib/api.js index 8bcd47f9a..e72c92cac 100644 --- a/lib/api.js +++ b/lib/api.js @@ -20,6 +20,7 @@ import {observeWorkerProcess} from './plugin-support/shared-workers.js'; import RunStatus from './run-status.js'; import scheduler from './scheduler.js'; import serializeError from './serialize-error.js'; +import {fileOrderSeed, generateSeed, shuffle} from './test-order.js'; function normalizeRequireOption(require) { return arrify(require).map(name => { @@ -92,6 +93,7 @@ export default class Api extends Emittery { let setupOrGlobError; const apiOptions = this.options; + const randomSeed = apiOptions.randomize ? (apiOptions.randomSeed ?? generateSeed()) : undefined; // Each run will have its own status. It can only be created when test files // have been found. @@ -195,7 +197,13 @@ export default class Api extends Emittery { runStatus = new RunStatus(selectedFiles.length, null, selectionInsights); } - selectedFiles = scheduler.failingTestsFirst(selectedFiles, this._getLocalCacheDir(), this.options.cacheEnabled); + if (!randomSeed) { + selectedFiles = scheduler.failingTestsFirst(selectedFiles, this._getLocalCacheDir(), this.options.cacheEnabled); + } + + if (randomSeed) { + selectedFiles = shuffle(selectedFiles, fileOrderSeed(randomSeed)); + } const debugWithoutSpecificFile = Boolean(this.options.debug) && !this.options.debug.active && selectedFiles.length !== 1; @@ -208,6 +216,7 @@ export default class Api extends Emittery { matching: apiOptions.match.length > 0 || runtimeOptions.interactiveMatchPattern !== undefined, previousFailures: runtimeOptions.countPreviousFailures?.() ?? 0, firstRun: runtimeOptions.firstRun ?? true, + randomSeed, status: runStatus, }); @@ -274,6 +283,7 @@ export default class Api extends Emittery { ...forkOptions, providerStates, lineNumbers, + randomSeed, recordNewSnapshots: !isCi, match: runtimeOptions.interactiveMatchPattern === undefined ? match : [...match, runtimeOptions.interactiveMatchPattern], }; diff --git a/lib/cli.js b/lib/cli.js index f62887755..0c88822a2 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -54,6 +54,16 @@ const FLAGS = { description: 'Additional Node.js arguments for launching worker processes (specify as a single string)', type: 'string', }, + randomize: { + coerce: coerceLastValue, + description: 'Randomize test file and test order', + type: 'boolean', + }, + seed: { + coerce: coerceLastValue, + description: 'Seed for randomized test order', + type: 'string', + }, serial: { alias: 's', coerce: coerceLastValue, @@ -321,6 +331,16 @@ export default async function loadCli() { // eslint-disable-line complexity exit('’sortTestFiles’ must be a comparator function.'); } + const hasSeed = Object.hasOwn(combined, 'seed'); + const randomize = combined.randomize === true || (combined.randomize !== false && hasSeed); + let randomSeed; + if (randomize && hasSeed) { + randomSeed = String(combined.seed); + if (randomSeed.length === 0) { + exit('The --seed flag must be provided with a non-empty value.'); + } + } + if (Object.hasOwn(conf, 'watch')) { exit('’watch’ must not be configured, use the --watch CLI flag instead.'); } @@ -419,6 +439,8 @@ export default async function loadCli() { // eslint-disable-line complexity match, nodeArguments, parallelRuns, + randomize, + randomSeed, sortTestFiles: conf.sortTestFiles, projectDir, providers, diff --git a/lib/reporters/default.js b/lib/reporters/default.js index 1b5035968..f9ef96657 100644 --- a/lib/reporters/default.js +++ b/lib/reporters/default.js @@ -157,6 +157,9 @@ export default class Reporter { } this.lineWriter.writeLine(); + if (plan.randomSeed) { + this.lineWriter.writeLine(colors.information(`Random seed: ${plan.randomSeed}`)); + } } consumeStateChange(event) { // eslint-disable-line complexity diff --git a/lib/reporters/tap.js b/lib/reporters/tap.js index 7884db59f..13610cab8 100644 --- a/lib/reporters/tap.js +++ b/lib/reporters/tap.js @@ -72,6 +72,9 @@ export default class TapReporter { plan.status.on('stateChange', ({data: evt}) => this.consumeStateChange(evt)); this.reportStream.write(supertap.start() + os.EOL); + if (plan.randomSeed) { + this.reportStream.write(`# Random seed: ${plan.randomSeed}${os.EOL}`); + } } endRun() { diff --git a/lib/runner.js b/lib/runner.js index e8c219b81..07b4b4539 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -9,6 +9,7 @@ import createChain from './create-chain.js'; import parseTestArgs from './parse-test-args.js'; import serializeError from './serialize-error.js'; import {load as loadSnapshots, determineSnapshotDir} from './snapshot-manager.js'; +import {shuffle, testOrderSeed} from './test-order.js'; import Runnable from './test.js'; import {waitForReady} from './worker/state.js'; @@ -33,6 +34,7 @@ export default class Runner extends Emittery { this.checkSelectedByLineNumbers = options.checkSelectedByLineNumbers; this.matchPatterns = options.match ?? []; this.projectDir = options.projectDir; + this.randomSeed = options.randomSeed; this.recordNewSnapshots = options.recordNewSnapshots === true; this.serial = options.serial === true; this.snapshotDir = options.snapshotDir; @@ -512,7 +514,8 @@ export default class Runner extends Emittery { // If a concurrent test fails, even if `failFast` is enabled it won't // stop other concurrent tests from running. - const allOkays = await Promise.all(concurrentTests.map(task => this.runTest(task, contextRef.copy()))); + const testOrder = this.randomSeed ? shuffle(concurrentTests, testOrderSeed(this.randomSeed, this.file)) : concurrentTests; + const allOkays = await Promise.all(testOrder.map(task => this.runTest(task, contextRef.copy()))); return allOkays.every(Boolean); }); diff --git a/lib/test-order.js b/lib/test-order.js new file mode 100644 index 000000000..5f94ebb0b --- /dev/null +++ b/lib/test-order.js @@ -0,0 +1,45 @@ +import crypto from 'node:crypto'; + +export function generateSeed() { + return crypto.randomBytes(8).toString('hex'); +} + +export function fileOrderSeed(seed) { + return `files:${seed}`; +} + +export function testOrderSeed(seed, file) { + return `tests:${seed}:${file}`; +} + +const modulus = 2_147_483_647; +const multiplier = 48_271; + +function createRandom(seed) { + let state = 1; + + for (const character of seed) { + state = ((state * 31) + character.codePointAt(0)) % modulus; + } + + if (state === 0) { + state = 1; + } + + return () => { + state = (state * multiplier) % modulus; + return (state - 1) / (modulus - 1); + }; +} + +export function shuffle(items, seed) { + const shuffled = [...items]; + const random = createRandom(seed); + + for (let index = shuffled.length - 1; index > 0; index--) { + const swapIndex = Math.floor(random() * (index + 1)); + [shuffled[index], shuffled[swapIndex]] = [shuffled[swapIndex], shuffled[index]]; + } + + return shuffled; +} diff --git a/lib/worker/base.js b/lib/worker/base.js index 7a5de284a..984986900 100644 --- a/lib/worker/base.js +++ b/lib/worker/base.js @@ -79,6 +79,7 @@ const run = async options => { file: options.file, match: options.match, projectDir: options.projectDir, + randomSeed: options.randomSeed, recordNewSnapshots: options.recordNewSnapshots, serial: options.serial, snapshotDir: options.snapshotDir, diff --git a/test-tap/fixture/randomize-tests/package.json b/test-tap/fixture/randomize-tests/package.json new file mode 100644 index 000000000..bedb411a9 --- /dev/null +++ b/test-tap/fixture/randomize-tests/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/test-tap/fixture/randomize-tests/test.js b/test-tap/fixture/randomize-tests/test.js new file mode 100644 index 000000000..5761ce5aa --- /dev/null +++ b/test-tap/fixture/randomize-tests/test.js @@ -0,0 +1,21 @@ +import test from '../../../entrypoints/main.js'; + +test('alpha', t => { + t.pass(); +}); + +test('bravo', t => { + t.pass(); +}); + +test('charlie', t => { + t.pass(); +}); + +test('delta', t => { + t.pass(); +}); + +test('echo', t => { + t.pass(); +}); diff --git a/test-tap/integration/assorted.js b/test-tap/integration/assorted.js index 00c5c868c..fc946f72f 100644 --- a/test-tap/integration/assorted.js +++ b/test-tap/integration/assorted.js @@ -7,6 +7,7 @@ import {stripVTControlCharacters} from 'node:util'; import ciInfo from 'ci-info'; import {test} from 'tap'; +import {shuffle, testOrderSeed} from '../../lib/test-order.js'; import {execCli} from '../helper/cli.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -160,3 +161,62 @@ test('uses sortTestFiles to sort test files', t => { t.end(); }); }); + +test('--seed reproduces file order even when failed-test cache changes', t => { + const fixtureDir = path.join(__dirname, '..', 'fixture', 'sort-tests'); + const cacheDir = path.join(fixtureDir, 'node_modules', '.cache', 'ava'); + const cacheFile = path.join(cacheDir, 'failing-tests.json'); + const file0 = path.join(fixtureDir, '0.js'); + const file1 = path.join(fixtureDir, '1.js'); + + const runWithCache = failedFile => new Promise((resolve, reject) => { + fs.mkdirSync(cacheDir, {recursive: true}); + fs.writeFileSync(cacheFile, JSON.stringify([failedFile])); + + execCli(['--tap', '--seed=ava-seed'], { + dirname: 'fixture/sort-tests', + env: {AVA_FORCE_CI: 'not-ci'}, + }, (error, stdout) => { + if (error) { + reject(error); + return; + } + + resolve([...stdout.matchAll(/^ok \d+ - (\d+) ›/gm)].map(([, file]) => file)); + }); + }); + + runWithCache(file0) + .then(firstOrder => runWithCache(file1).then(secondOrder => [firstOrder, secondOrder])) + .then(([firstOrder, secondOrder]) => { + t.strictSame(firstOrder, secondOrder); + t.end(); + }, error => { + t.error(error); + t.end(); + }); +}); + +test('--seed randomizes test order and reports the seed', t => { + const seed = 'ava-seed'; + const testFile = path.join(__dirname, '..', 'fixture', 'randomize-tests', 'test.js'); + const titles = ['alpha', 'bravo', 'charlie', 'delta', 'echo']; + const expectedTitles = shuffle(titles, testOrderSeed(seed, testFile)); + + t.notSame(expectedTitles, titles); + + execCli(['--seed=ava-seed', 'randomize-tests/test.js'], (error, stdout) => { + t.error(error); + t.match(stdout, /Random seed: ava-seed/); + t.match(stdout, new RegExp(expectedTitles.join(String.raw`[\s\S]+?`))); + t.end(); + }); +}); + +test('--randomize reports a generated seed', t => { + execCli(['--randomize', 'randomize-tests/test.js'], (error, stdout) => { + t.error(error); + t.match(stdout, /Random seed: [\da-f]{16}/); + t.end(); + }); +}); diff --git a/test-tap/runner.js b/test-tap/runner.js index 72d03832c..68d262bda 100644 --- a/test-tap/runner.js +++ b/test-tap/runner.js @@ -3,6 +3,7 @@ import {setTimeout as delay} from 'node:timers/promises'; import {test} from 'tap'; import Runner from '../lib/runner.js'; +import {shuffle, testOrderSeed} from '../lib/test-order.js'; import {set as setOptions} from '../lib/worker/options.js'; setOptions({}); @@ -99,6 +100,33 @@ test('run serial tests before concurrent ones', t => { }); }); +test('options.randomSeed randomizes concurrent tests but preserves serial tests', t => { + const array = []; + const serialTitles = ['serial 1', 'serial 2']; + const concurrentTitles = ['alpha', 'bravo', 'charlie', 'delta', 'echo']; + const expectedConcurrentTitles = shuffle(concurrentTitles, testOrderSeed('ava-seed', import.meta.url)); + + t.notSame(expectedConcurrentTitles, concurrentTitles); + + return promiseEnd(new Runner({file: import.meta.url, randomSeed: 'ava-seed'}), runner => { + for (const title of serialTitles) { + runner.chain.serial(title, a => { + array.push(title); + a.pass(); + }); + } + + for (const title of concurrentTitles) { + runner.chain(title, a => { + array.push(title); + a.pass(); + }); + } + }).then(() => { + t.strictSame(array, [...serialTitles, ...expectedConcurrentTitles]); + }); +}); + test('anything can be skipped', t => { const array = []; function pusher(title) {