test: make the generator and dev server tests stable

test/create-webpack-app/test.utils.js

@@ -1,5 +1,6 @@
 "use strict";
 
+const os = require("node:os");
 const path = require("node:path");
 const testUtilsPkg = require("../utils/test-utils");
 
@@ -8,6 +9,35 @@ const nodeVersion = Number.parseInt(process.versions.node.split(".")[0], 10);
 // eslint-disable-next-line jsdoc/reject-any-type
 /** @typedef {Record<string, any>} TestOptions */
 
+// The generators install real packages. Skip the requests the tests don't need, so fewer
+// requests can fail. `prefer-offline` isn't used: with a stale cache, npm misses new
+// versions that other packages already depend on.
+const packageManagerEnv = {
+  npm_config_audit: "false",
+  npm_config_fund: "false",
+  npm_config_update_notifier: "false",
+  // Yarn 1 breaks its cache when two installs fill it at the same time, as tests in parallel
+  // Jest workers do, so each worker gets its own
+  YARN_CACHE_FOLDER: path.join(
+    os.tmpdir(),
+    `webpack-cli-yarn-cache-${process.env.JEST_WORKER_ID || 1}`,
+  ),
+};
+
+/**
+ * Fail with the package manager's error, rather than with the output the test misses.
+ * @template {{ stderr: string }} T
+ * @param {T} result result of a CLI run
+ * @returns {T} the same result
+ */
+const checkInstall = (result) => {
+  if (result.stderr && result.stderr.includes("Error occurred while installing packages")) {
+    throw new Error(`Installing packages failed:\n${result.stderr}`);
+  }
+
+  return result;
+};
+
 function createPathDependentUtils(cli) {
   const CLI_PATH = path.resolve(__dirname, `../../packages/${cli}/bin/cli.js`);
 
@@ -19,10 +49,13 @@ function createPathDependentUtils(cli) {
    * @returns {Promise<import("execa").Result>} child process
    */
   const run = async (cwd, args = [], options = {}) =>
-    testUtilsPkg.run(cwd, args, {
-      ...options,
-      executorPath: CLI_PATH,
-    });
+    checkInstall(
+      await testUtilsPkg.run(cwd, args, {
+        ...options,
+        env: { ...packageManagerEnv, ...options.env },
+        executorPath: CLI_PATH,
+      }),
+    );
 
   /**
    * Run the webpack CLI in watch mode for a test case.
@@ -34,6 +67,7 @@ function createPathDependentUtils(cli) {
   const runWatch = async (cwd, args = [], options = {}) =>
     testUtilsPkg.runWatch(cwd, args, {
       ...options,
+      env: { ...packageManagerEnv, ...options.env },
       executorPath: CLI_PATH,
     });
 
@@ -46,10 +80,13 @@ function createPathDependentUtils(cli) {
    * @returns {Promise<{ stdout: string, stderr: string }>} result
    */
   const runPromptWithAnswers = async (cwd, args, answers = [], options = {}) =>
-    testUtilsPkg.runPromptWithAnswers(cwd, args, answers, {
-      ...options,
-      executorPath: CLI_PATH,
-    });
+    checkInstall(
+      await testUtilsPkg.runPromptWithAnswers(cwd, args, answers, {
+        ...options,
+        env: { ...packageManagerEnv, ...options.env },
+        executorPath: CLI_PATH,
+      }),
+    );
 
   return { run, runWatch, runPromptWithAnswers };
 }

test/serve/basic/multi-dev-server.config.js

@@ -1,11 +1,10 @@
+const { getFreePort } = require("../../utils/test-utils");
 const WebpackCLITestPlugin = require("../../utils/webpack-cli-test-plugin");
 const { devServerConfig } = require("./helper/base-dev-server.config");
 
-const getGetPort = () => import("get-port");
-
 module.exports = async () => {
-  const port1 = await (await getGetPort()).default();
-  const port2 = await (await getGetPort()).default();
+  const port1 = await getFreePort();
+  const port2 = await getFreePort();
 
   return [
     {

test/serve/basic/serve-basic.test.js

@@ -1,9 +1,12 @@
 "use strict";
 
 const path = require("node:path");
-const { normalizeStderr, normalizeStdout, runWatch } = require("../../utils/test-utils");
-
-const getGetPort = () => import("get-port");
+const {
+  getFreePort,
+  normalizeStderr,
+  normalizeStdout,
+  runWatch,
+} = require("../../utils/test-utils");
 
 const testPath = path.resolve(__dirname);
 
@@ -16,7 +19,7 @@ describe("basic serve usage", () => {
   let port;
 
   beforeEach(async () => {
-    port = await (await getGetPort()).default();
+    port = await getFreePort();
   });
 
   it("should work", async () => {

test/serve/serve-variable/serve-variable.test.js

@@ -1,17 +1,15 @@
 "use strict";
 
 const path = require("node:path");
-const { normalizeStderr, runWatch } = require("../../utils/test-utils");
-
-const getGetPort = () => import("get-port");
+const { getFreePort, normalizeStderr, runWatch } = require("../../utils/test-utils");
 
 const testPath = path.resolve(__dirname);
 
 describe("serve variable", () => {
   let port;
 
   beforeEach(async () => {
-    port = await (await getGetPort()).default();
+    port = await getFreePort();
   });
 
   it("compiles without flags and export variable", async () => {

test/serve/with-custom-port/serve-custom-config.test.js

@@ -1,17 +1,15 @@
 "use strict";
 
 const path = require("node:path");
-const { normalizeStderr, runWatch } = require("../../utils/test-utils");
-
-const getGetPort = () => import("get-port");
+const { getFreePort, normalizeStderr, runWatch } = require("../../utils/test-utils");
 
 const testPath = path.resolve(__dirname);
 
 describe("serve with devServer in config", () => {
   let port;
 
   beforeEach(async () => {
-    port = await (await getGetPort()).default();
+    port = await getFreePort();
   });
 
   it("should pick up the host and port from config", async () => {

test/utils/test-utils.js

@@ -401,6 +401,23 @@ const cleanStderr = (stderr) =>
     ? stderr.replaceAll(NODE_WARNING_RE, "").replaceAll(INFRASTRUCTURE_INFO_RE, "").trim()
     : stderr;
 
+/**
+ * Get a free port for a dev server.
+ *
+ * A port from the OS comes from the ephemeral range, where outgoing connections (e.g. of
+ * tests that install packages) can take it before the dev server listens, and Jest
+ * workers running in parallel can get the same one. So each worker takes ports from its
+ * own range below the ephemeral ones.
+ * @returns {Promise<number>} free port
+ */
+const getFreePort = async () => {
+  const { default: getPort, portNumbers } = await import("get-port");
+  // 120 ranges keep ports below 32768, where the ephemeral range of Linux starts
+  const from = 20000 + ((Number(process.env.JEST_WORKER_ID || 1) - 1) % 120) * 100;
+
+  return getPort({ port: portNumbers(from, from + 99) });
+};
+
 const uniqueDirectoryForTest = async () => {
   const result = path.resolve(path.resolve(__dirname, "../create-webpack-app-testing"), uuid());
 
@@ -413,6 +430,7 @@ const uniqueDirectoryForTest = async () => {
 
 module.exports = {
   cleanStderr,
+  getFreePort,
   getWebpackCliArguments,
   hyphenToUpperCase,
   isWindows,