ci: run all unit test files from last to first (temporary, do not merge)

Order stress test for #342237: loads the test files in their usual order, then runs them from last to first, in one process and in parallel workers, on Linux, macOS and Windows.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: alexdima

.github/workflows/pr-darwin-test.yml

@@ -149,17 +149,21 @@ jobs:
             sleep 5
           done
 
+      # TEMPORARY (order stress test): run the test files from last to first, in one process or in parallel workers
       - name: 🧪 Run unit tests (Electron)
         if: ${{ inputs.electron_tests && inputs.unit_tests }}
-        timeout-minutes: 15
-        run: ./scripts/test.sh --tfs "Unit Tests" --workers
+        timeout-minutes: 30
+        run: ./scripts/test.sh --tfs "Unit Tests" ${{ contains(inputs.job_name, 'Parallel') && '--workers' || '' }}
         env:
+          VSCODE_TEST_REVERSE_FILES: '1'
           VSCODE_SKIP_PRELAUNCH: '1'
 
       - name: 🧪 Run unit tests (node.js)
-        if: ${{ inputs.electron_tests && inputs.unit_tests }}
-        timeout-minutes: 15
-        run: npm run test-node -- --workers
+        if: ${{ !cancelled() && inputs.electron_tests && inputs.unit_tests }}
+        timeout-minutes: 30
+        run: npm run test-node -- ${{ contains(inputs.job_name, 'Parallel') && '--workers' || '' }}
+        env:
+          VSCODE_TEST_REVERSE_FILES: '1'
 
       - name: 🧪 Run unit tests (Browser, Webkit)
         if: ${{ inputs.browser_tests && inputs.unit_tests }}

.github/workflows/pr-linux-test.yml

@@ -301,18 +301,22 @@ jobs:
             sleep 5
           done
 
+      # TEMPORARY (order stress test): run the test files from last to first, in one process or in parallel workers
       - name: 🧪 Run unit tests (Electron)
         if: ${{ inputs.electron_tests && inputs.unit_tests }}
-        timeout-minutes: 15
-        run: ./scripts/test.sh --tfs "Unit Tests" --workers
+        timeout-minutes: 30
+        run: ./scripts/test.sh --tfs "Unit Tests" ${{ contains(inputs.job_name, 'Parallel') && '--workers' || '' }}
         env:
+          VSCODE_TEST_REVERSE_FILES: '1'
           DISPLAY: ":10"
           VSCODE_SKIP_PRELAUNCH: '1'
 
       - name: 🧪 Run unit tests (node.js)
-        if: ${{ inputs.electron_tests && inputs.unit_tests }}
-        timeout-minutes: 15
-        run: npm run test-node -- --workers
+        if: ${{ !cancelled() && inputs.electron_tests && inputs.unit_tests }}
+        timeout-minutes: 30
+        run: npm run test-node -- ${{ contains(inputs.job_name, 'Parallel') && '--workers' || '' }}
+        env:
+          VSCODE_TEST_REVERSE_FILES: '1'
 
       - name: 🧪 Run unit tests (Browser, Chromium)
         if: ${{ inputs.browser_tests && inputs.unit_tests }}

.github/workflows/pr-win32-test.yml

@@ -187,19 +187,23 @@ jobs:
             }
           }
 
+      # TEMPORARY (order stress test): run the test files from last to first, in one process or in parallel workers
       - name: 🧪 Run unit tests (Electron)
         if: ${{ inputs.electron_tests && inputs.unit_tests }}
-        timeout-minutes: 15
+        timeout-minutes: 30
         shell: pwsh
-        run: .\scripts\test.bat --tfs "Unit Tests" --workers
+        run: .\scripts\test.bat --tfs "Unit Tests" ${{ contains(inputs.job_name, 'Parallel') && '--workers' || '' }}
         env:
+          VSCODE_TEST_REVERSE_FILES: '1'
           VSCODE_SKIP_PRELAUNCH: '1'
 
       - name: 🧪 Run unit tests (node.js)
-        if: ${{ inputs.electron_tests && inputs.unit_tests }}
-        timeout-minutes: 15
+        if: ${{ !cancelled() && inputs.electron_tests && inputs.unit_tests }}
+        timeout-minutes: 30
         shell: pwsh
-        run: npm run test-node -- --workers
+        run: npm run test-node -- ${{ contains(inputs.job_name, 'Parallel') && '--workers' || '' }}
+        env:
+          VSCODE_TEST_REVERSE_FILES: '1'
 
       - name: 🧪 Run unit tests (Browser, Chromium)
         if: ${{ inputs.browser_tests && inputs.unit_tests }}

.github/workflows/tmp-unit-reverse-order.yml

@@ -0,0 +1,67 @@
+name: Unit tests from last to first (temporary, do not merge)
+
+# TEMPORARY (order stress test): runs all Electron and node.js unit test files from last to first,
+# in one process (Serial) and in parallel worker processes (Parallel), on every platform.
+
+on:
+  push:
+    branches:
+      - agents/tmp-unit-reverse-order
+
+permissions:
+  contents: read
+
+jobs:
+  linux-serial:
+    name: Linux
+    uses: ./.github/workflows/pr-linux-test.yml
+    with:
+      job_name: Reverse-Serial
+      electron_tests: true
+      integration_tests: false
+      smoke_tests: false
+
+  linux-parallel:
+    name: Linux
+    uses: ./.github/workflows/pr-linux-test.yml
+    with:
+      job_name: Reverse-Parallel
+      electron_tests: true
+      integration_tests: false
+      smoke_tests: false
+
+  macos-serial:
+    name: macOS
+    uses: ./.github/workflows/pr-darwin-test.yml
+    with:
+      job_name: Reverse-Serial
+      electron_tests: true
+      integration_tests: false
+      smoke_tests: false
+
+  macos-parallel:
+    name: macOS
+    uses: ./.github/workflows/pr-darwin-test.yml
+    with:
+      job_name: Reverse-Parallel
+      electron_tests: true
+      integration_tests: false
+      smoke_tests: false
+
+  windows-serial:
+    name: Windows
+    uses: ./.github/workflows/pr-win32-test.yml
+    with:
+      job_name: Reverse-Serial
+      electron_tests: true
+      integration_tests: false
+      smoke_tests: false
+
+  windows-parallel:
+    name: Windows
+    uses: ./.github/workflows/pr-win32-test.yml
+    with:
+      job_name: Reverse-Parallel
+      electron_tests: true
+      integration_tests: false
+      smoke_tests: false

test/unit/electron/index.js

@@ -351,6 +351,9 @@ app.on('ready', async () => {
 		}
 	});
 
+	// TEMPORARY (order stress test): proves in the log that the test files run from last to first
+	ipcMain.on('tmp-order-log', (_, message) => console.log(message));
+
 	// We need to provide a basic `ISandboxConfiguration`
 	// for our preload script to function properly because
 	// some of our types depend on it (e.g. product.ts).

test/unit/electron/renderer.js

@@ -66,7 +66,7 @@ const path = require('path');
 const glob = require('glob');
 const util = require('util');
 const coverage = require('../coverage');
-const { trackTestFiles, distributeTestFiles } = require('../parallel.ts');
+const { trackTestFiles, distributeTestFiles, reverseTestFiles } = require('../parallel.ts');
 const { pathToFileURL } = require('url');
 
 // Disabled custom inspect. See #38847
@@ -462,11 +462,17 @@ async function runTests(opts) {
 		mocha.timeout(opts.timeout);
 	}
 
-	const testFiles = opts.workerDir ? trackTestFiles(mocha.suite) : undefined;
+	// TEMPORARY (order stress test): VSCODE_TEST_REVERSE_FILES runs the test files from last to first
+	const reverse = !!process.env.VSCODE_TEST_REVERSE_FILES;
+	let testFiles = opts.workerDir || reverse ? trackTestFiles(mocha.suite) : undefined;
 
 	await loadTests(opts);
 
-	if (testFiles) {
+	if (testFiles && reverse) {
+		testFiles = reverseTestFiles(mocha.suite, testFiles);
+		ipcRenderer.send('tmp-order-log', `TEMPORARY: running ${testFiles.size} test files from last to first, starting with ${[...testFiles.keys()][0]}`);
+	}
+	if (testFiles && opts.workerDir) {
 		distributeTestFiles(mocha.suite, testFiles, { workerId: opts.worker, workerDir: opts.workerDir });
 	}
 

test/unit/node/index.js

@@ -20,7 +20,7 @@ import { fileURLToPath, pathToFileURL } from 'url';
 import semver from 'semver';
 import parallel from '../parallel.ts';
 
-const { getWorkerCount, removeArgument, runInWorkers, trackTestFiles, distributeTestFiles } = parallel;
+const { getWorkerCount, removeArgument, runInWorkers, trackTestFiles, distributeTestFiles, reverseTestFiles } = parallel;
 
 /**
  * @type {{ build: boolean; run: string; runGlob: string; coverage: boolean; help: boolean; coverageFormats: string | string[]; coveragePath: string; timeout?: string; workers?: string; worker?: string; workerDir?: string; }}
@@ -161,7 +161,9 @@ function main() {
 		timeout: args.timeout ?? (IS_CI ? 30000 : 5000)
 	});
 
-	const testFiles = args.workerDir ? trackTestFiles(runner.suite) : undefined;
+	// TEMPORARY (order stress test): VSCODE_TEST_REVERSE_FILES runs the test files from last to first
+	const reverse = !!process.env.VSCODE_TEST_REVERSE_FILES;
+	let testFiles = args.workerDir || reverse ? trackTestFiles(runner.suite) : undefined;
 
 	/**
 	 * @param modules
@@ -257,7 +259,11 @@ function main() {
 				unexpectedErrors.push((err && err.message ? err.message : err) + '\n' + stack);
 			});
 
-			if (testFiles) {
+			if (testFiles && reverse) {
+				testFiles = reverseTestFiles(runner.suite, testFiles);
+				console.log(`TEMPORARY: running ${testFiles.size} test files from last to first, starting with ${[...testFiles.keys()][0]}`);
+			}
+			if (testFiles && args.workerDir) {
 				distributeTestFiles(runner.suite, testFiles, { workerId: args.worker ?? '', workerDir: args.workerDir ?? '' });
 			}
 

test/unit/parallel.ts

@@ -247,6 +247,23 @@ function distributeTestFiles(root: Suite, files: Map<string, ITestFile>, { worke
 	});
 }
 
+/**
+ * TEMPORARY (order stress test): makes the test files run from last to first. They are still
+ * loaded in their usual order, so only the order in which they run changes.
+ * @returns the files in reverse order, which is also the order in which workers claim them
+ */
+function reverseTestFiles(root: Suite, files: Map<string, ITestFile>): Map<string, ITestFile> {
+	const reversed = new Map([...files].reverse());
+	const fileSuites = new Set([...files.values()].flatMap(file => file.suites));
+	const fileTests = new Set([...files.values()].flatMap(file => file.tests));
+	const firstFileSuite = root.suites.findIndex(suite => fileSuites.has(suite));
+	const otherSuitesBefore = root.suites.filter((suite, i) => !fileSuites.has(suite) && (firstFileSuite === -1 || i < firstFileSuite));
+	const otherSuitesAfter = root.suites.filter((suite, i) => !fileSuites.has(suite) && firstFileSuite !== -1 && i > firstFileSuite);
+	root.suites = [...otherSuitesBefore, ...[...reversed.values()].flatMap(file => file.suites), ...otherSuitesAfter];
+	root.tests = [...root.tests.filter(test => !fileTests.has(test)), ...[...reversed.values()].flatMap(file => file.tests)];
+	return reversed;
+}
+
 /**
  * @returns a promise that resolves once the data is flushed, so exiting does not truncate it
  */
@@ -259,4 +276,4 @@ function formatDuration(ms: number): string {
 	return seconds >= 60 ? `${Math.floor(seconds / 60)}m ${seconds % 60}s` : `${seconds}s`;
 }
 
-module.exports = { getWorkerCount, removeArgument, runInWorkers, trackTestFiles, distributeTestFiles };
+module.exports = { getWorkerCount, removeArgument, runInWorkers, trackTestFiles, distributeTestFiles, reverseTestFiles };