[codex] Preserve seeded file-order reproducibility when randomizing test runs

docs/05-command-line.md

@@ -37,6 +37,8 @@ Options:
       --no-worker-threads  Don't use worker threads                    [boolean]
       --node-arguments     Additional Node.js arguments for launching worker
                            processes (specify as a single string)       [string]
+      --randomize          Randomize test file and test order          [boolean]
+      --seed               Seed for randomized test order               [string]
   -s, --serial             Run tests serially                          [boolean]
   -t, --tap                Generate TAP output                         [boolean]
   -T, --timeout            Set global timeout (milliseconds or human-readable,
@@ -80,6 +82,22 @@ Files inside `node_modules` are *always* ignored. So are files starting with `_`
 
 When using `npm test`, you can pass positional arguments directly `npm test test2.js`, but flags needs to be passed like `npm test -- --verbose`.
 
+## Randomizing test order
+
+Use the `--randomize` flag to run test files, and concurrent tests within each file, in a random order. AVA reports the seed used for each run:
+
+```console
+npx ava --randomize
+```
+
+Use `--seed` to reproduce the same order:
+
+```console
+npx ava --seed=4f7a2c91
+```
+
+Serial tests still run in declaration order.
+
 ## Running tests with matching titles
 
 [![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/avajs/ava/tree/main/examples/matching-titles?file=test.js&terminal=test&view=editor)

docs/06-configuration.md

@@ -51,6 +51,8 @@ Arguments passed to the CLI will always take precedence over the CLI options con
 - `failFast`: stop running further tests once a test fails
 - `failWithoutAssertions`: if `false`, does not fail a test if it doesn't run [assertions](./03-assertions.md)
 - `environmentVariables`: specifies environment variables to be made available to the tests. The environment variables defined here override the ones from `process.env`
+- `randomize`: if `true`, randomizes test file order and concurrent test order within each file. Serial tests still run in declaration order
+- `seed`: sets the seed used for randomized test order. This implies `randomize`, unless `randomize` is `false`
 - `serial`: if `true`, prevents parallel execution of tests within a file
 - `tap`: if `true`, enables the [TAP reporter](./05-command-line.md#tap-reporter)
 - `verbose`: if `true`, enables verbose output (though there currently non-verbose output is not supported)

lib/api.js

@@ -20,6 +20,7 @@ import {observeWorkerProcess} from './plugin-support/shared-workers.js';
 import RunStatus from './run-status.js';
 import scheduler from './scheduler.js';
 import serializeError from './serialize-error.js';
+import {fileOrderSeed, generateSeed, shuffle} from './test-order.js';
 
 function normalizeRequireOption(require) {
 	return arrify(require).map(name => {
@@ -92,6 +93,7 @@ export default class Api extends Emittery {
 		let setupOrGlobError;
 
 		const apiOptions = this.options;
+		const randomSeed = apiOptions.randomize ? (apiOptions.randomSeed ?? generateSeed()) : undefined;
 
 		// Each run will have its own status. It can only be created when test files
 		// have been found.
@@ -195,7 +197,13 @@ export default class Api extends Emittery {
 				runStatus = new RunStatus(selectedFiles.length, null, selectionInsights);
 			}
 
-			selectedFiles = scheduler.failingTestsFirst(selectedFiles, this._getLocalCacheDir(), this.options.cacheEnabled);
+			if (!randomSeed) {
+				selectedFiles = scheduler.failingTestsFirst(selectedFiles, this._getLocalCacheDir(), this.options.cacheEnabled);
+			}
+
+			if (randomSeed) {
+				selectedFiles = shuffle(selectedFiles, fileOrderSeed(randomSeed));
+			}
 
 			const debugWithoutSpecificFile = Boolean(this.options.debug) && !this.options.debug.active && selectedFiles.length !== 1;
 
@@ -208,6 +216,7 @@ export default class Api extends Emittery {
 				matching: apiOptions.match.length > 0 || runtimeOptions.interactiveMatchPattern !== undefined,
 				previousFailures: runtimeOptions.countPreviousFailures?.() ?? 0,
 				firstRun: runtimeOptions.firstRun ?? true,
+				randomSeed,
 				status: runStatus,
 			});
 
@@ -274,6 +283,7 @@ export default class Api extends Emittery {
 					...forkOptions,
 					providerStates,
 					lineNumbers,
+					randomSeed,
 					recordNewSnapshots: !isCi,
 					match: runtimeOptions.interactiveMatchPattern === undefined ? match : [...match, runtimeOptions.interactiveMatchPattern],
 				};

lib/cli.js

@@ -54,6 +54,16 @@ const FLAGS = {
 		description: 'Additional Node.js arguments for launching worker processes (specify as a single string)',
 		type: 'string',
 	},
+	randomize: {
+		coerce: coerceLastValue,
+		description: 'Randomize test file and test order',
+		type: 'boolean',
+	},
+	seed: {
+		coerce: coerceLastValue,
+		description: 'Seed for randomized test order',
+		type: 'string',
+	},
 	serial: {
 		alias: 's',
 		coerce: coerceLastValue,
@@ -321,6 +331,16 @@ export default async function loadCli() { // eslint-disable-line complexity
 		exit('’sortTestFiles’ must be a comparator function.');
 	}
 
+	const hasSeed = Object.hasOwn(combined, 'seed');
+	const randomize = combined.randomize === true || (combined.randomize !== false && hasSeed);
+	let randomSeed;
+	if (randomize && hasSeed) {
+		randomSeed = String(combined.seed);
+		if (randomSeed.length === 0) {
+			exit('The --seed flag must be provided with a non-empty value.');
+		}
+	}
+
 	if (Object.hasOwn(conf, 'watch')) {
 		exit('’watch’ must not be configured, use the --watch CLI flag instead.');
 	}
@@ -419,6 +439,8 @@ export default async function loadCli() { // eslint-disable-line complexity
 		match,
 		nodeArguments,
 		parallelRuns,
+		randomize,
+		randomSeed,
 		sortTestFiles: conf.sortTestFiles,
 		projectDir,
 		providers,

lib/reporters/default.js

@@ -157,6 +157,9 @@ export default class Reporter {
 		}
 
 		this.lineWriter.writeLine();
+		if (plan.randomSeed) {
+			this.lineWriter.writeLine(colors.information(`Random seed: ${plan.randomSeed}`));
+		}
 	}
 
 	consumeStateChange(event) { // eslint-disable-line complexity

lib/reporters/tap.js

@@ -72,6 +72,9 @@ export default class TapReporter {
 		plan.status.on('stateChange', ({data: evt}) => this.consumeStateChange(evt));
 
 		this.reportStream.write(supertap.start() + os.EOL);
+		if (plan.randomSeed) {
+			this.reportStream.write(`# Random seed: ${plan.randomSeed}${os.EOL}`);
+		}
 	}
 
 	endRun() {

lib/runner.js

@@ -9,6 +9,7 @@ import createChain from './create-chain.js';
 import parseTestArgs from './parse-test-args.js';
 import serializeError from './serialize-error.js';
 import {load as loadSnapshots, determineSnapshotDir} from './snapshot-manager.js';
+import {shuffle, testOrderSeed} from './test-order.js';
 import Runnable from './test.js';
 import {waitForReady} from './worker/state.js';
 
@@ -33,6 +34,7 @@ export default class Runner extends Emittery {
 		this.checkSelectedByLineNumbers = options.checkSelectedByLineNumbers;
 		this.matchPatterns = options.match ?? [];
 		this.projectDir = options.projectDir;
+		this.randomSeed = options.randomSeed;
 		this.recordNewSnapshots = options.recordNewSnapshots === true;
 		this.serial = options.serial === true;
 		this.snapshotDir = options.snapshotDir;
@@ -512,7 +514,8 @@ export default class Runner extends Emittery {
 
 			// If a concurrent test fails, even if `failFast` is enabled it won't
 			// stop other concurrent tests from running.
-			const allOkays = await Promise.all(concurrentTests.map(task => this.runTest(task, contextRef.copy())));
+			const testOrder = this.randomSeed ? shuffle(concurrentTests, testOrderSeed(this.randomSeed, this.file)) : concurrentTests;
+			const allOkays = await Promise.all(testOrder.map(task => this.runTest(task, contextRef.copy())));
 			return allOkays.every(Boolean);
 		});
 

lib/test-order.js

@@ -0,0 +1,45 @@
+import crypto from 'node:crypto';
+
+export function generateSeed() {
+	return crypto.randomBytes(8).toString('hex');
+}
+
+export function fileOrderSeed(seed) {
+	return `files:${seed}`;
+}
+
+export function testOrderSeed(seed, file) {
+	return `tests:${seed}:${file}`;
+}
+
+const modulus = 2_147_483_647;
+const multiplier = 48_271;
+
+function createRandom(seed) {
+	let state = 1;
+
+	for (const character of seed) {
+		state = ((state * 31) + character.codePointAt(0)) % modulus;
+	}
+
+	if (state === 0) {
+		state = 1;
+	}
+
+	return () => {
+		state = (state * multiplier) % modulus;
+		return (state - 1) / (modulus - 1);
+	};
+}
+
+export function shuffle(items, seed) {
+	const shuffled = [...items];
+	const random = createRandom(seed);
+
+	for (let index = shuffled.length - 1; index > 0; index--) {
+		const swapIndex = Math.floor(random() * (index + 1));
+		[shuffled[index], shuffled[swapIndex]] = [shuffled[swapIndex], shuffled[index]];
+	}
+
+	return shuffled;
+}

lib/worker/base.js

@@ -79,6 +79,7 @@ const run = async options => {
 		file: options.file,
 		match: options.match,
 		projectDir: options.projectDir,
+		randomSeed: options.randomSeed,
 		recordNewSnapshots: options.recordNewSnapshots,
 		serial: options.serial,
 		snapshotDir: options.snapshotDir,

test-tap/fixture/randomize-tests/package.json

@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}

test-tap/fixture/randomize-tests/test.js

@@ -0,0 +1,21 @@
+import test from '../../../entrypoints/main.js';
+
+test('alpha', t => {
+	t.pass();
+});
+
+test('bravo', t => {
+	t.pass();
+});
+
+test('charlie', t => {
+	t.pass();
+});
+
+test('delta', t => {
+	t.pass();
+});
+
+test('echo', t => {
+	t.pass();
+});

test-tap/integration/assorted.js

@@ -7,6 +7,7 @@ import {stripVTControlCharacters} from 'node:util';
 import ciInfo from 'ci-info';
 import {test} from 'tap';
 
+import {shuffle, testOrderSeed} from '../../lib/test-order.js';
 import {execCli} from '../helper/cli.js';
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url));
@@ -160,3 +161,62 @@ test('uses sortTestFiles to sort test files', t => {
 		t.end();
 	});
 });
+
+test('--seed reproduces file order even when failed-test cache changes', t => {
+	const fixtureDir = path.join(__dirname, '..', 'fixture', 'sort-tests');
+	const cacheDir = path.join(fixtureDir, 'node_modules', '.cache', 'ava');
+	const cacheFile = path.join(cacheDir, 'failing-tests.json');
+	const file0 = path.join(fixtureDir, '0.js');
+	const file1 = path.join(fixtureDir, '1.js');
+
+	const runWithCache = failedFile => new Promise((resolve, reject) => {
+		fs.mkdirSync(cacheDir, {recursive: true});
+		fs.writeFileSync(cacheFile, JSON.stringify([failedFile]));
+
+		execCli(['--tap', '--seed=ava-seed'], {
+			dirname: 'fixture/sort-tests',
+			env: {AVA_FORCE_CI: 'not-ci'},
+		}, (error, stdout) => {
+			if (error) {
+				reject(error);
+				return;
+			}
+
+			resolve([...stdout.matchAll(/^ok \d+ - (\d+) ›/gm)].map(([, file]) => file));
+		});
+	});
+
+	runWithCache(file0)
+		.then(firstOrder => runWithCache(file1).then(secondOrder => [firstOrder, secondOrder]))
+		.then(([firstOrder, secondOrder]) => {
+			t.strictSame(firstOrder, secondOrder);
+			t.end();
+		}, error => {
+			t.error(error);
+			t.end();
+		});
+});
+
+test('--seed randomizes test order and reports the seed', t => {
+	const seed = 'ava-seed';
+	const testFile = path.join(__dirname, '..', 'fixture', 'randomize-tests', 'test.js');
+	const titles = ['alpha', 'bravo', 'charlie', 'delta', 'echo'];
+	const expectedTitles = shuffle(titles, testOrderSeed(seed, testFile));
+
+	t.notSame(expectedTitles, titles);
+
+	execCli(['--seed=ava-seed', 'randomize-tests/test.js'], (error, stdout) => {
+		t.error(error);
+		t.match(stdout, /Random seed: ava-seed/);
+		t.match(stdout, new RegExp(expectedTitles.join(String.raw`[\s\S]+?`)));
+		t.end();
+	});
+});
+
+test('--randomize reports a generated seed', t => {
+	execCli(['--randomize', 'randomize-tests/test.js'], (error, stdout) => {
+		t.error(error);
+		t.match(stdout, /Random seed: [\da-f]{16}/);
+		t.end();
+	});
+});

test-tap/runner.js

@@ -3,6 +3,7 @@ import {setTimeout as delay} from 'node:timers/promises';
 import {test} from 'tap';
 
 import Runner from '../lib/runner.js';
+import {shuffle, testOrderSeed} from '../lib/test-order.js';
 import {set as setOptions} from '../lib/worker/options.js';
 
 setOptions({});
@@ -99,6 +100,33 @@ test('run serial tests before concurrent ones', t => {
 	});
 });
 
+test('options.randomSeed randomizes concurrent tests but preserves serial tests', t => {
+	const array = [];
+	const serialTitles = ['serial 1', 'serial 2'];
+	const concurrentTitles = ['alpha', 'bravo', 'charlie', 'delta', 'echo'];
+	const expectedConcurrentTitles = shuffle(concurrentTitles, testOrderSeed('ava-seed', import.meta.url));
+
+	t.notSame(expectedConcurrentTitles, concurrentTitles);
+
+	return promiseEnd(new Runner({file: import.meta.url, randomSeed: 'ava-seed'}), runner => {
+		for (const title of serialTitles) {
+			runner.chain.serial(title, a => {
+				array.push(title);
+				a.pass();
+			});
+		}
+
+		for (const title of concurrentTitles) {
+			runner.chain(title, a => {
+				array.push(title);
+				a.pass();
+			});
+		}
+	}).then(() => {
+		t.strictSame(array, [...serialTitles, ...expectedConcurrentTitles]);
+	});
+});
+
 test('anything can be skipped', t => {
 	const array = [];
 	function pusher(title) {