perf(electron): build the Next standalone once and hydrate natives per leg (#10321 stage 8)

The desktop release matrix ran the full Next.js standalone build on all four legs (windows, macos-intel, macos-arm64, linux), duplicating the platform-neutral majority of that work four times and re-exposing every leg to the hosted-runner RAM class of failure that took the linux leg out of v3.8.49.

- scripts/build/standaloneTarball.mjs: deterministic, dependency-free tar.gz writer/reader (uid/gid/mtime pinned, sorted entries, symlink + exec-bit preservation; GNU-tar interop covered by tests).
- scripts/build/standaloneManifest.mjs: byte-level manifest of .build/next (sha256 + size + symlink target per entry, plus the archive's own digest) catching artifact-transfer corruption before extraction and re-verifying the restored tree byte-for-byte, smuggling included.
- scripts/build/standaloneBundle.mjs: pack / restore / hydrate CLI over the two modules above.
- scripts/build/hydrateNativeDeps.mjs: swaps install-machine-forked native optionals (@img/sharp-*, @ngrok/ngrok-*, fsevents) from the leg's own npm ci into the restored tree, then verifies the bundled-native closure (koffi triplets, better-sqlite3 prebuilds, wreq-js, onnxruntime with its documented darwin-x64 exemption) services the leg's platform/arch before packaging starts.
- .github/workflows/electron-release.yml: new web-build job builds the standalone once on ubuntu with webpack and uploads the bundle; legs download, restore, and hydrate it, skipping the per-leg build. The legacy per-leg build remains as a rollback path via the ELECTRON_SHARED_STANDALONE workflow_dispatch input, and legs fail closed if web-build ran and failed.

Regression tests cover archive roundtrip, byte determinism, manifest tamper/smuggle detection, forked-native swaps, and native-closure serviceability.

Author: backryun

.github/workflows/electron-release.yml

@@ -55,9 +55,75 @@ jobs:
           echo "version=$VERSION" >> "$GITHUB_OUTPUT"
           echo "✓ Valid version: $VERSION"
 
+  web-build:
+    name: Build shared Next standalone
+    needs: validate
+    # Stage 8 (issue #10321): the four desktop legs used to each run the full
+    # `npm run build` (Next standalone) — ~111 runner-minutes per release just to
+    # produce the same platform-independent bundle four times. This job builds it
+    # once on ubuntu; every leg then restores the byte-verified archive and
+    # re-forks its native optionals (scripts/build/standaloneBundle.mjs).
+    #
+    # Rollback lever: set the repo variable ELECTRON_SHARED_STANDALONE=disabled.
+    # This job then skips, every leg falls back to building its own web bundle
+    # (the legacy step below), and the pipeline behaves exactly like pre-Stage 8 —
+    # no revert needed.
+    if: ${{ !cancelled() && needs.validate.result == 'success' && vars.ELECTRON_SHARED_STANDALONE != 'disabled' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: Setup Node
+        uses: actions/setup-node@v7
+        with:
+          node-version: 24
+
+      - name: Install dependencies
+        run: npm ci
+        env:
+          NPM_CONFIG_LEGACY_PEER_DEPS: true
+
+      - name: Build Next.js standalone
+        # webpack, not Turbopack, for the same hosted-runner RAM reason as the
+        # linux leg (see the long comment on the fallback step in `build`).
+        env:
+          JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
+          NODE_OPTIONS: "--max_old_space_size=6144"
+          OMNIROUTE_USE_TURBOPACK: "0"
+        run: npm run build
+
+      - name: Pack standalone bundle
+        # Deterministic tar.gz + byte-level manifest; the manifest embeds the
+        # archive's own sha256 so artifact-transfer corruption is caught before
+        # extraction, and every entry is re-verified after extraction.
+        run: node scripts/build/standaloneBundle.mjs pack --out web-bundle.tar.gz
+
+      - name: Upload shared web bundle
+        uses: actions/upload-artifact@v7
+        with:
+          name: web-standalone-bundle
+          # compression-level 0: the payload is already a deterministic tar.gz;
+          # re-zipping would only burn runner CPU without shrinking it further.
+          compression-level: 0
+          # Legs consume this within minutes; no reason to retain it like the
+          # installer artifacts (default 90d).
+          retention-days: 3
+          path: |
+            web-bundle.tar.gz
+            web-bundle.tar.gz.manifest.json
+
   build:
     name: Build Electron (${{ matrix.platform }})
-    needs: validate
+    needs: [validate, web-build]
+    # `web-build` is skipped when ELECTRON_SHARED_STANDALONE=disabled (rollback
+    # mode); legs then run the legacy per-leg web build below. If it ran and
+    # failed, fail closed: legs cannot package without the bundle, and silently
+    # falling back to four per-leg builds would hide exactly the regression the
+    # shared job exists to surface.
+    if: ${{ !cancelled() && needs.validate.result == 'success' && (needs.web-build.result == 'success' || needs.web-build.result == 'skipped') }}
     runs-on: ${{ matrix.runner }}
     permissions:
       contents: write # electron-builder may publish artifacts with GH_TOKEN
@@ -69,19 +135,27 @@ jobs:
             runner: windows-latest
             target: win
             ext: .exe
+            os: win32
+            arch: x64
           - platform: macos-intel
             runner: macos-15-intel
             target: mac-x64
             ext: .dmg
+            os: darwin
+            arch: x64
           - platform: macos-arm64
             runner: macos-latest
             target: mac-arm64
             ext: -arm64.dmg
+            os: darwin
+            arch: arm64
           - platform: linux
             runner: ubuntu-latest
             target: linux
             ext: .AppImage
             deb_ext: .deb
+            os: linux
+            arch: x64,arm64
 
     steps:
       - uses: actions/checkout@v7
@@ -116,7 +190,11 @@ jobs:
           mkdir -p "$RUNNER_TEMP/home"
           echo "USERPROFILE=$RUNNER_TEMP/home" >> "$GITHUB_ENV"
 
-      - name: Build Next.js standalone
+      - name: Build Next.js standalone (legacy per-leg fallback)
+        # Stage 8: only runs in rollback mode (ELECTRON_SHARED_STANDALONE=disabled)
+        # or when the shared web-build job was skipped. Otherwise the leg restores
+        # the shared bundle from the `web-build` job below.
+        if: needs.web-build.result == 'skipped'
         env:
           JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
           NODE_OPTIONS: "--max_old_space_size=6144"
@@ -134,6 +212,30 @@ jobs:
           OMNIROUTE_USE_TURBOPACK: ${{ matrix.platform == 'linux' && '0' || '1' }}
         run: npm run build
 
+      - name: Download shared web bundle
+        # Stage 8: inverse of the fallback step above — runs exactly when the
+        # shared `web-build` job produced the bundle.
+        if: needs.web-build.result == 'success'
+        uses: actions/download-artifact@v8
+        with:
+          name: web-standalone-bundle
+
+      - name: Restore + hydrate shared web bundle
+        if: needs.web-build.result == 'success'
+        shell: bash
+        # restore: verify the archive's sha256 against the manifest, extract, then
+        #          re-verify every entry (existence + size + content hash + symlink
+        #          targets, and no unlisted files) byte-for-byte.
+        # hydrate: the bundle was built on ubuntu, so install-machine-forked native
+        #          optionals (@img/sharp-*, @img/sharp-libvips-*, @ngrok/ngrok-*,
+        #          fsevents) carry linux forks. Replace them with the forks this
+        #          leg's own `npm ci` resolved, then assert every bundled native
+        #          (koffi triplets, better-sqlite3 prebuilds, wreq-js, onnxruntime)
+        #          can service this leg's platform/arch before packaging starts.
+        run: |
+          node scripts/build/standaloneBundle.mjs restore --archive web-bundle.tar.gz
+          node scripts/build/standaloneBundle.mjs hydrate --platform ${{ matrix.os }} --arch ${{ matrix.arch }}
+
       - name: Sync version in electron/package.json
         shell: bash
         env:

scripts/build/hydrateNativeDeps.mjs

@@ -0,0 +1,137 @@
+#!/usr/bin/env node
+/**
+ * Platform hydration for the shared Next standalone web build (issue #10321,
+ * Stage 8).
+ *
+ * The standalone bundle is built ONCE on ubuntu and restored on every desktop
+ * matrix leg. Everything except install-machine-forked optional packages is
+ * platform-independent:
+ *
+ *  - Bundled-for-all (verify only): koffi ships every triplet under
+ *    `build/koffi/<os>_<arch>`, better-sqlite3 v13 ships Node-API prebuilds for
+ *    8 platforms, wreq-js ships `rust/wreq-js.<plat>-<arch>[-libc].node`, and
+ *    onnxruntime-node ships `bin/napi-v6/<os>/<arch>`.
+ *  - Install-machine-forked (hydrate): `@img/sharp-*`, `@img/sharp-libvips-*`,
+ *    `@ngrok/ngrok-*` and macOS-only `fsevents` resolve to whichever platform
+ *    ran `npm ci`. The ubuntu-built tree carries the linux forks; each leg
+ *    replaces them with the forks from its OWN `npm ci`d node_modules.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+/** Scope prefixes whose members are install-machine-forked. */
+export const HYDRATED_SCOPES = ["@img/sharp-", "@img/sharp-libvips-", "@ngrok/ngrok-"];
+
+/** Standalone packages that are not forked but must never be platform-forked. */
+export const HYDRATED_ROOT_PACKAGES = ["fsevents"];
+
+/**
+ * onnxruntime-node does not publish a darwin-x64 binary for napi-v6 (only
+ * linux/win32 x64 + darwin arm64), so existence cannot be asserted there.
+ */
+export const BUNDLED_EXEMPTIONS = new Set(["onnxruntime-node:darwin-x64"]);
+
+function platformTriple(platform, arch) {
+  // koffi uses underscore triplets; better-sqlite3/wreq-js/onnx use dashes.
+  return { koffi: `${platform}_${arch}`, dash: `${platform}-${arch}` };
+}
+
+function rmrf(target) {
+  fs.rmSync(target, { recursive: true, force: true });
+}
+
+function copyDir(from, to) {
+  fs.cpSync(from, to, { recursive: true, verbatimSymlinks: false, force: true });
+}
+
+function directMemberNames(nodeModulesDir, scope) {
+  const scopeDir = path.join(nodeModulesDir, ...scope.split("/").slice(0, -1));
+  const prefix = scope.split("/").pop();
+  try {
+    return fs
+      .readdirSync(scopeDir)
+      .filter((name) => name.startsWith(prefix))
+      .map((name) => `${scope.slice(0, scope.lastIndexOf("/"))}/${name}`);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Replace install-machine-forked packages inside the restored standalone tree
+ * with the forks resolved by THIS machine's node_modules.
+ *
+ * @param {{standaloneNodeModules: string, sourceNodeModules: string}} opts
+ * @returns {{replaced: string[], removed: string[], copied: string[]}}
+ */
+export function hydratePlatformNatives({ standaloneNodeModules, sourceNodeModules }) {
+  const replaced = [];
+  const removed = [];
+  const copied = [];
+
+  const forkedNames = new Set();
+  for (const scope of HYDRATED_SCOPES) {
+    for (const name of directMemberNames(sourceNodeModules, scope)) forkedNames.add(name);
+    for (const name of directMemberNames(standaloneNodeModules, scope)) forkedNames.add(name);
+  }
+  for (const pkg of HYDRATED_ROOT_PACKAGES) {
+    if (fs.existsSync(path.join(sourceNodeModules, pkg))) forkedNames.add(pkg);
+    if (fs.existsSync(path.join(standaloneNodeModules, pkg))) forkedNames.add(pkg);
+  }
+
+  for (const name of forkedNames) {
+    const standalonePath = path.join(standaloneNodeModules, ...name.split("/"));
+    const sourcePath = path.join(sourceNodeModules, ...name.split("/"));
+    const hadIt = fs.existsSync(standalonePath);
+    const hasIt = fs.existsSync(sourcePath);
+    if (hadIt) rmrf(standalonePath);
+    if (!hasIt) {
+      if (hadIt) removed.push(name);
+      continue; // e.g. fsevents on non-darwin legs: simply absent everywhere.
+    }
+    copyDir(sourcePath, standalonePath);
+    copied.push(name);
+    if (hadIt) replaced.push(name);
+  }
+  return { replaced, removed, copied };
+}
+
+/**
+ * Assert that every bundled native dependency can service `platform`/`arch`.
+ *
+ * @returns {{ok: true} | {ok: false, errors: string[]}}
+ */
+export function verifyBundledNatives({ nodeModulesDir, platform, arch }) {
+  const errors = [];
+  const triple = platformTriple(platform, arch);
+
+  const koffiDir = path.join(nodeModulesDir, "koffi", "build", "koffi", triple.koffi);
+  if (!fs.existsSync(koffiDir)) errors.push(`koffi: missing bundled triplet ${triple.koffi}`);
+
+  const sqlitePrebuild = path.join(
+    nodeModulesDir,
+    "better-sqlite3",
+    "prebuilds",
+    `${triple.dash}.node`
+  );
+  if (!fs.existsSync(sqlitePrebuild))
+    errors.push(`better-sqlite3: missing prebuild ${triple.dash}.node`);
+
+  const wreqDir = path.join(nodeModulesDir, "wreq-js", "rust");
+  const wreqNames = fs.existsSync(wreqDir)
+    ? fs
+        .readdirSync(wreqDir)
+        .filter((n) => n.startsWith(`wreq-js.${triple.dash}`) && n.endsWith(".node"))
+    : [];
+  if (wreqNames.length === 0) errors.push(`wreq-js: missing rust binary for ${triple.dash}`);
+
+  const exempt = BUNDLED_EXEMPTIONS.has(`onnxruntime-node:${triple.dash}`);
+  if (!exempt) {
+    const onnxDir = path.join(nodeModulesDir, "onnxruntime-node", "bin", "napi-v6", platform, arch);
+    if (!fs.existsSync(onnxDir))
+      errors.push(`onnxruntime-node: missing ${platform}/${arch} binary`);
+  }
+
+  return errors.length === 0 ? { ok: true } : { ok: false, errors };
+}