bench: compare pnpm engines on Next.js

benchmarks/benchmarkFixture.js

@@ -107,7 +107,7 @@ export default async function benchmark (pm, fixture, opts) {
     // up with the cache/ directory the rest of the flow cleans between scenarios.
     env.PNPM_HOME = path.join(cwd, 'cache')
   }
-  cpSync(path.join(FIXTURES_DIR, fixture), cwd, { recursive: true })
+  cpSync(opts.fixtureDir ?? path.join(FIXTURES_DIR, fixture), cwd, { recursive: true })
   const modules = opts.hasNodeModules ? path.join(cwd, 'node_modules') : null
 
   cleanLockfile(pm, cwd, env)
@@ -254,4 +254,3 @@ function spawnSyncOrThrow (cmd, opts) {
   }
   return result;
 }
-

benchmarks/checkoutNextFixture.js

@@ -0,0 +1,65 @@
+import fs from 'fs'
+import path from 'path'
+import spawn from 'cross-spawn'
+
+const NEXT_COMMIT = '31b78bbed91c538e6cf196faad0a37afdfae7e70'
+
+export default async function checkoutNextFixture (fixtureDir) {
+  await fs.promises.mkdir(fixtureDir, { recursive: true })
+  runGit(fixtureDir, ['init'])
+  runGit(fixtureDir, ['remote', 'add', 'origin', 'https://github.com/vercel/next.js.git'])
+  runGit(fixtureDir, ['sparse-checkout', 'init', '--no-cone'])
+  runGit(fixtureDir, [
+    'sparse-checkout',
+    'set',
+    '--no-cone',
+    '/package.json',
+    '/pnpm-lock.yaml',
+    '/pnpm-workspace.yaml',
+    '/.npmrc',
+    '/patches/',
+    '**/package.json',
+  ])
+  runGit(fixtureDir, ['fetch', '--depth=1', '--filter=blob:none', 'origin', NEXT_COMMIT])
+  runGit(fixtureDir, ['checkout', '--detach', 'FETCH_HEAD'])
+  await fs.promises.rm(`${fixtureDir}/.git`, { recursive: true })
+  await removeEmptyDirectories(fixtureDir)
+  await migrateSettings(fixtureDir)
+}
+
+function runGit (cwd, args) {
+  const result = spawn.sync('git', args, { cwd, stdio: 'inherit' })
+  if (result.status !== 0) {
+    throw new Error(`git ${args.join(' ')} failed with exit code ${result.status}`)
+  }
+}
+
+async function removeEmptyDirectories (dir) {
+  const entries = await fs.promises.readdir(dir, { withFileTypes: true })
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      await removeEmptyDirectories(`${dir}/${entry.name}`)
+    }
+  }
+  if ((await fs.promises.readdir(dir)).length === 0) {
+    await fs.promises.rmdir(dir)
+  }
+}
+
+async function migrateSettings (fixtureDir) {
+  const manifestPath = path.join(fixtureDir, 'package.json')
+  const workspacePath = path.join(fixtureDir, 'pnpm-workspace.yaml')
+  const manifest = JSON.parse(await fs.promises.readFile(manifestPath, 'utf8'))
+  const settings = manifest.pnpm
+  delete manifest.pnpm
+  await fs.promises.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`)
+  await fs.promises.appendFile(workspacePath, [
+    '',
+    'linkWorkspacePackages: true',
+    `overrides: ${JSON.stringify(settings.overrides)}`,
+    `packageExtensions: ${JSON.stringify(settings.packageExtensions)}`,
+    `patchedDependencies: ${JSON.stringify(settings.patchedDependencies)}`,
+    'strictDepBuilds: false',
+    '',
+  ].join('\n'))
+}

benchmarks/index.js

@@ -7,6 +7,7 @@ import prettyMs from 'pretty-ms'
 import tempy from 'tempy'
 import cmdsMap from './commandsMap.js'
 import benchmark from './recordBenchmark.js'
+import checkoutNextFixture from './checkoutNextFixture.js'
 import generateSvg from './generateSvg.js'
 import generateStackedSvg from './generateStackedSvg.js'
 import spawn from "cross-spawn"
@@ -40,6 +41,11 @@ const fixtures = [
   {
     name: 'alotta-files',
     mdDesc: '## Lots of Files\n\nThe app\'s `package.json` [here](https://github.com/pnpm/pnpm.io/blob/main/benchmarks/fixtures/alotta-files/package.json)'
+  },
+  {
+    name: 'nextjs',
+    mdDesc: '## Next.js Monorepo\n\nThis fixture uses [vercel/next.js at commit `31b78bb`](https://github.com/vercel/next.js/tree/31b78bbed91c538e6cf196faad0a37afdfae7e70), with its pnpm settings migrated to `pnpm-workspace.yaml`. It reproduces the workload from [pnpm/pnpm issue 13305](https://github.com/pnpm/pnpm/issues/13305) and compares the TypeScript and Rust pnpm engines on a large multi-project workspace with thousands of registry tarballs.',
+    pnpmOnly: true,
   }
 ]
 
@@ -102,6 +108,8 @@ run()
 
 async function run () {
   const tmpDir = tempy.directory()
+  const nextFixtureDir = path.join(tmpDir, 'fixtures', 'nextjs')
+  await checkoutNextFixture(nextFixtureDir)
   const managersDirs = {}
   for (const pm of ['npm', 'pnpm11', 'pnpm12', 'yarn']) {
     managersDirs[pm] = path.join(tmpDir, pm)
@@ -128,7 +136,6 @@ async function run () {
     { key: 'yarn', managersDir: managersDirs.yarn },
     { key: 'yarn_pnp', managersDir: managersDirs.yarn, hasNodeModules: false },
   ]
-  const pms = pmConfigs.map(({ key }) => key)
   const tableRows = [
     { test: 'firstInstall',               action: 'install', cache: ' ',   lockfile: ' ',   nodeModules: ' '   },
     { test: 'withWarmModules',            action: 'install', cache: ' ',   lockfile: ' ',   nodeModules: '✔',   needsNodeModules: true },
@@ -144,22 +151,27 @@ async function run () {
   const svgs = []
   let sortedTests = tests
   for (const fixture of fixtures) {
+    const fixturePmConfigs = fixture.pnpmOnly
+      ? pmConfigs.filter(({ key }) => key.startsWith('pnpm'))
+      : pmConfigs
+    const fixturePms = fixturePmConfigs.map(({ key }) => key)
     const results = {}
-    for (const { key, managersDir, hasNodeModules } of pmConfigs) {
+    for (const { key, managersDir, hasNodeModules } of fixturePmConfigs) {
       results[key] = min(await benchmark(cmdsMap[key], fixture.name, {
         limitRuns: LIMIT_RUNS,
         hasNodeModules: hasNodeModules ?? true,
         managersDir,
+        fixtureDir: fixture.name === 'nextjs' ? nextFixtureDir : undefined,
       }))
     }
-    sortedTests = sortTestsBySlowest(tests, results, pms)
+    sortedTests = sortTestsBySlowest(tests, results, fixturePms)
     const sortedDescriptions = sortedTests.map(t => testDescriptions[t])
     const sortedTableRows = sortedTests.map(t => tableRows.find(r => r.test === t))
 
-    const headerLegends = pms.map(pm => cmdsMap[pm].mdLegend ?? cmdsMap[pm].legend).join(' | ')
-    const headerSep = pms.map(() => '---').join(' | ')
+    const headerLegends = fixturePms.map(pm => cmdsMap[pm].mdLegend ?? cmdsMap[pm].legend).join(' | ')
+    const headerSep = fixturePms.map(() => '---').join(' | ')
     const rows = sortedTableRows.map(({ test, action, cache, lockfile, nodeModules, needsNodeModules }) => {
-      const values = pmConfigs.map(({ key, hasNodeModules: pmHasNodeModules }) => {
+      const values = fixturePmConfigs.map(({ key, hasNodeModules: pmHasNodeModules }) => {
         if (needsNodeModules && pmHasNodeModules === false) return 'n/a'
         return prettyMs(results[key][test])
       }).join(' | ')
@@ -168,48 +180,50 @@ async function run () {
 
     // Main chart: pnpm 11 and pnpm 12 are merged into a single stacked bar so
     // pnpm 12's speedup over pnpm 11 is visible at a glance.
-    const mainBars = [
-      { ...cmdsMap.npm, key: 'npm' },
-      {
-        stacked: true,
-        color: cmdsMap.pnpm12.color,
-        legend: cmdsMap.pnpm12.legend,
-        displayVersion: cmdsMap.pnpm12.displayVersion,
-        extraColor: '#cccccc',
-        extraLegend: 'pnpm 11 extra',
-        primaryKey: 'pnpm12',
-        secondaryKey: 'pnpm11',
-      },
-      { ...cmdsMap.yarn, key: 'yarn' },
-      { ...cmdsMap.yarn_pnp, key: 'yarn_pnp' },
-    ]
-    const resArray = sortedTests.map(test => mainBars.map(bar => bar.stacked
-      ? {
-          primary: Math.round(results[bar.primaryKey][test] / 100) / 10,
-          secondary: Math.round(results[bar.secondaryKey][test] / 100) / 10,
-        }
-      : Math.round(results[bar.key][test] / 100) / 10
-    ))
-    const mainSvg = generateSvg(resArray, mainBars, sortedDescriptions, formattedNow)
-    const mainSvgHash = hashContent(mainSvg)
-    sections.push(stripIndents`
-      ${fixture.mdDesc}
-
-      | action  | cache | lockfile | node_modules| ${headerLegends} |
-      | ---     | ---   | ---      | ---         | ${headerSep} |
-      ${rows}
-
-      <img alt="Graph of the ${fixture.name} results" src="/img/benchmarks/${fixture.name}.svg?v=${mainSvgHash}" />
-    `)
-
-    svgs.push({
-      path: path.join(BENCH_IMGS, `${fixture.name}.svg`),
-      file: mainSvg
-    })
+    if (!fixture.pnpmOnly) {
+      const mainBars = [
+        { ...cmdsMap.npm, key: 'npm' },
+        {
+          stacked: true,
+          color: cmdsMap.pnpm12.color,
+          legend: cmdsMap.pnpm12.legend,
+          displayVersion: cmdsMap.pnpm12.displayVersion,
+          extraColor: '#cccccc',
+          extraLegend: 'pnpm 11 extra',
+          primaryKey: 'pnpm12',
+          secondaryKey: 'pnpm11',
+        },
+        { ...cmdsMap.yarn, key: 'yarn' },
+        { ...cmdsMap.yarn_pnp, key: 'yarn_pnp' },
+      ]
+      const resArray = sortedTests.map(test => mainBars.map(bar => bar.stacked
+        ? {
+            primary: Math.round(results[bar.primaryKey][test] / 100) / 10,
+            secondary: Math.round(results[bar.secondaryKey][test] / 100) / 10,
+          }
+        : Math.round(results[bar.key][test] / 100) / 10
+      ))
+      const mainSvg = generateSvg(resArray, mainBars, sortedDescriptions, formattedNow)
+      const mainSvgHash = hashContent(mainSvg)
+      sections.push(stripIndents`
+        ${fixture.mdDesc}
+
+        | action  | cache | lockfile | node_modules| ${headerLegends} |
+        | ---     | ---   | ---      | ---         | ${headerSep} |
+        ${rows}
+
+        <img alt="Graph of the ${fixture.name} results" src="/img/benchmarks/${fixture.name}.svg?v=${mainSvgHash}" />
+      `)
+
+      svgs.push({
+        path: path.join(BENCH_IMGS, `${fixture.name}.svg`),
+        file: mainSvg
+      })
+    }
 
     // pnpm-only comparison: include only scenarios that every selected pnpm version supports.
     // Sorted independently of the main chart, keyed by pnpm 11 (the first pnpm config).
-    const pnpmConfigs = pmConfigs.filter(({ key }) => key.startsWith('pnpm'))
+    const pnpmConfigs = fixturePmConfigs.filter(({ key }) => key.startsWith('pnpm'))
     const pnpmKeys = pnpmConfigs.map(({ key }) => key)
     const pnpmSortedTests = sortTestsBySlowest(tests, results, pnpmKeys)
       .filter((test) => {

benchmarks/recordBenchmark.js

@@ -20,6 +20,7 @@ export default async function (pm, fixture, opts) {
   if (prevResults.length >= limitRuns) return prevResults
 
   const newResults = await benchmark(pm, fixture, {
+    fixtureDir: opts.fixtureDir,
     hasNodeModules: opts.hasNodeModules,
     managersDir: opts.managersDir,
   })