refactor: make all.html a composed page of the Node.js preset

all.html, every module page's content on one page, is Node.js's, but
html built it in, behind `generateAllPage`, and the streaming work
spread its bookkeeping through generate.mjs.

html now takes `composedPages`: modules default-exporting a page's
metadata and which pages are its parts. It assembles each from its
parts' compiled content in sidebar order, compiles it once they all
are, and renders it first, all in utils/composed.mjs. all.html moves to
@node-core/doc-kit as one of them, which the Node.js preset lists, and
`generateAllPage` goes away. Built with the preset, the Node.js API
docs come out byte-identical, all.html included.

Assisted-by: Claude Opus 5.5 <noreply@anthropic.com>

Author: ovflowd

.changeset/composed-pages.md

@@ -0,0 +1,6 @@
+---
+'@doc-kit/generator-react': minor
+'@node-core/doc-kit': minor
+---
+
+The `html` generator's `generateAllPage` option is replaced by `composedPages`: modules default-exporting a page made of other pages' content, `{ head, isPart }`, which `html` assembles from those pages' compiled content, in sidebar order, once they all are. `all.html` is now one of them, added by the Node.js preset (`@node-core/doc-kit/config`), so sites built without the preset no longer get one.

.changeset/hold-less-in-memory.md

@@ -4,4 +4,4 @@
 '@doc-kit/generator-react': patch
 ---
 
-Builds hold less in memory. The nodes of `metadata`'s entries no longer keep their `position` in the source, so neither does the output of `json-simple`, now about a third of its size. Each chunk of a streamed output is let go once every generator reading it has read that chunk, rather than once they have all read all of it, and `runGenerators(configuration, { results: false })` keeps none of the requested generators' output, which the CLI now uses. `all.html` renders one module at a time, rather than building every module's elements first.
+Builds hold less in memory. The nodes of `metadata`'s entries no longer keep their `position` in the source, so neither does the output of `json-simple`, now about a third of its size. Each chunk of a streamed output is let go once every generator reading it has read that chunk, rather than once they have all read all of it, and `runGenerators(configuration, { results: false })` keeps none of the requested generators' output, which the CLI now uses. A composed page, such as the Node.js preset's `all.html`, renders one part at a time, rather than building every part's elements first.

e2e/doc-kit.config.mjs

@@ -3,5 +3,10 @@ export default {
     // The announcement banner under test is opt-in: point it at the URL the
     // spec intercepts (no real request is made).
     remoteConfigUrl: 'https://nodejs.org/site.json',
+
+    // The navigation under test goes to the Node.js preset's `all.html`
+    composedPages: [
+      new URL('../packages/node/src/config/all-page.mjs', import.meta.url).href,
+    ],
   },
 };

packages/node/src/config/__tests__/index.test.mjs

@@ -1,7 +1,9 @@
 import assert from 'node:assert/strict';
 import { existsSync } from 'node:fs';
+import { join } from 'node:path';
 import { describe, it } from 'node:test';
 
+import allPage from '../all-page.mjs';
 import preset from '../index.mjs';
 
 describe('Node.js preset', () => {
@@ -22,4 +24,19 @@ describe('Node.js preset', () => {
   it('should ship the Node.js llms.txt template', () => {
     assert.ok(existsSync(preset['llms-txt'].templatePath));
   });
+
+  it('should compose all.html of the module pages', () => {
+    assert.deepEqual(preset.html.composedPages, [
+      join(import.meta.dirname, '..', 'all-page.mjs'),
+    ]);
+    assert.equal(allPage.head.api, 'all');
+
+    assert.equal(allPage.isPart({ api: 'fs' }), true);
+    assert.equal(allPage.isPart({ api: 'index' }), false);
+    assert.equal(allPage.isPart({ api: '404', synthetic: true }), false);
+    assert.equal(
+      allPage.isPart({ api: 'fs', chunk: { name: 'fs.promises' } }),
+      false
+    );
+  });
 });

packages/node/src/config/all-page.mjs

@@ -0,0 +1,31 @@
+'use strict';
+
+/**
+ * `all.html`: every module page's content on one page, in the order the
+ * sidebar lists them, made of the module pages rather than built again (see
+ * the `html` generator's `composedPages`).
+ */
+export default {
+  head: {
+    api: 'all',
+    path: '/all',
+    basename: 'all',
+    synthetic: true,
+    heading: {
+      type: 'heading',
+      depth: 1,
+      children: [{ type: 'text', value: 'All' }],
+      data: { name: 'All', text: 'All', slug: 'all' },
+    },
+  },
+
+  /**
+   * Whether a page is part of `all.html`: the module pages, minus the index.
+   * Chunk pages and the other synthetic pages are left out, as the module
+   * pages carry their content.
+   *
+   * @param {{ api: string, synthetic?: boolean, chunk?: object }} data
+   * @returns {boolean}
+   */
+  isPart: data => !data.synthetic && !data.chunk && data.api !== 'index',
+};

packages/node/src/config/index.mjs

@@ -70,6 +70,8 @@ export default {
 
     stylesheets: [join(import.meta.dirname, 'theme.css')],
 
+    composedPages: [join(import.meta.dirname, 'all-page.mjs')],
+
     imports: {
       '#theme/Logo': '@node-core/ui-components/Common/NodejsLogo',
     },

packages/react/src/html/README.md

@@ -13,8 +13,9 @@ library once and the client assets once, as soon as every page's metadata is
 known, and the worker pool imports, renders, templates, minifies and writes
 the pages one at a time as they are compiled, so memory scales with the largest
 page rather than with the site. With one thread, each step waits for the one
-before it to finish. `all.html` is assembled from the module pages' compiled
-content rather than built again from scratch.
+before it to finish. A composed page, such as the Node.js preset's
+`all.html`, is assembled from other pages' compiled content rather than built
+again from scratch.
 
 ## Configuring
 
@@ -50,10 +51,13 @@ content rather than built again from scratch.
   JSX-in-MDX. See [`components`](#components). **Default:** `{}`.
 - `navigation` {Object} Sidebar groups and navigation bar items. See
   [`navigation`](#navigation). **Default:** `{}`.
-- `generateAllPage` {boolean} When `true`, writes `all.html`: every module
-  page's content on one page, in sidebar order, assembled from the module pages
-  rather than built again. Chunk pages and the index are left out.
-  **Default:** `true`.
+- `composedPages` {Array} Pages made of other pages' content, one after the
+  other in sidebar order, with their tables of contents: modules (paths relative
+  to the working directory, or URLs) default-exporting `{ head, isPart }`, the
+  page's metadata and whether a page's metadata is one of its parts. Such a
+  page is compiled once all its parts are, and none is built again. The
+  Node.js preset adds `all.html`, every module page minus the index.
+  **Default:** `[]`.
 - `bundler` {WebBundler} Adapter that bundles the component library and the
   client assets, and compiles page programs. See
   [Bundler adapters](#bundler-adapters). **Default:** `createViteBundler()`.
@@ -482,7 +486,7 @@ export default ({ metadata }) => (
 - `headings` {Array} Pre-computed table of contents heading entries.
 - `readingTime` {string|undefined} Estimated reading time (e.g. `'5 min read'`).
   Only present when the `jsx-ast` generator's `showReadingTime` option is
-  enabled. `all.html` has none.
+  enabled. Composed pages have none.
 - `children` {ComponentChildren} Processed page content.
 
 The `Layout` component receives the props above. Custom Layout components can use

packages/react/src/html/__tests__/fixtures/all-page.mjs

@@ -0,0 +1,16 @@
+// A composed page of every module page, as the Node.js preset's `all.html`
+export default {
+  head: {
+    api: 'all',
+    path: '/all',
+    basename: 'all',
+    synthetic: true,
+    heading: {
+      type: 'heading',
+      depth: 1,
+      children: [{ type: 'text', value: 'All' }],
+      data: { name: 'All', text: 'All', slug: 'all' },
+    },
+  },
+  isPart: data => !data.synthetic && !data.chunk && data.api !== 'index',
+};

packages/react/src/html/__tests__/generate.test.mjs

@@ -29,6 +29,9 @@ async function* streamInOneBatch(items) {
   yield items;
 }
 
+// The Node.js preset's `all.html`, as a composed page
+const ALL_PAGE = import.meta.resolve('./fixtures/all-page.mjs');
+
 /**
  * Runs the generator over the given pages, streamed in as one batch, as
  * `jsx-ast` yields them.
@@ -105,9 +108,6 @@ const createTestConfiguration = async (context, target = ['html']) => {
     },
   });
 
-  // The pages under test are the module pages themselves
-  config.html.generateAllPage = false;
-
   return { config, output };
 };
 
@@ -253,7 +253,7 @@ describe('web generate', () => {
 
   it('assembles all.html from the module pages, in sidebar order', async context => {
     const { config, output } = await createTestConfiguration(context);
-    config.html.generateAllPage = true;
+    config.html.composedPages = [ALL_PAGE];
 
     const entries = [
       createEntry('zlib', 'Zlib'),
@@ -280,7 +280,7 @@ describe('web generate', () => {
 
   it('numbers the ids of all.html across its parts', async context => {
     const { config, output } = await createTestConfiguration(context);
-    config.html.generateAllPage = true;
+    config.html.composedPages = [ALL_PAGE];
 
     // Each module has code tabs, whose elements take ids
     const entries = ['zlib', 'fs'].map(api => {
@@ -556,7 +556,7 @@ describe('web generate', () => {
         const bundled = Promise.withResolvers();
 
         config.threads = 2;
-        config.html.generateAllPage = true;
+        config.html.composedPages = [ALL_PAGE];
         config.html.bundler = createStandInBundler(calls, { bundled });
 
         async function* input() {
@@ -601,7 +601,7 @@ describe('web generate', () => {
         const { config, output } = await createTestConfiguration(context);
 
         config.threads = threads;
-        config.html.generateAllPage = true;
+        config.html.composedPages = [ALL_PAGE];
         config.html.bundler = createStandInBundler([]);
 
         await generatePages(input);

packages/react/src/html/generate.mjs

@@ -10,10 +10,11 @@ import logger from '@doc-kit/core/logger/index.mjs';
 import { createChannel } from '@doc-kit/core/utils/channel.mjs';
 import getConfig from '@doc-kit/core/utils/configuration/index.mjs';
 import { toBatches } from '@doc-kit/core/utils/generators.mjs';
+import { importFromURL } from '@doc-kit/core/utils/loaders.mjs';
 
 import { headingSortFn } from '../jsx-ast/utils/getSortedHeadNodes.mjs';
 import { resolveBundler } from './bundlers/index.mjs';
-import { buildAllPage, createAllPageHead, isModulePage } from './utils/all.mjs';
+import { createComposedPages } from './utils/composed.mjs';
 import { copyStaticAssets } from './utils/copying.mjs';
 import createProgramBuilder, { moduleFileName } from './utils/generate.mjs';
 import { createPageWriter } from './utils/render.mjs';
@@ -28,8 +29,8 @@ const htmlLogger = logger.child('html');
  * page's program is compiled to a plain module as it arrives, the component
  * library and the client assets are bundled once every page's metadata is
  * known, and the worker pool renders the pages as they are compiled, one in
- * memory per worker. `all.html` imports the module pages' content, so it is
- * compiled once they all are. With one thread, the input arrives whole.
+ * memory per worker. A composed page (see `composedPages`) imports the content
+ * of the pages it is made of, so it is compiled once they all are. With one thread, the input arrives whole.
  *
  * @type {import('./types').Generator['generate']}
  */
@@ -138,7 +139,7 @@ const buildSite = async (input, options) => {
     const datas = [
       ...heads.filter(data => !data.synthetic).toSorted(headingSortFn),
       ...heads.filter(data => data.synthetic),
-      ...(config.generateAllPage ? [createAllPageHead()] : []),
+      ...composed.heads,
     ];
 
     // An empty directory of its own: the compiled pages are already in `outDir`
@@ -191,22 +192,11 @@ const buildSite = async (input, options) => {
 
   const compiled = [];
 
-  let allWritten = !config.generateAllPage;
+  const definitions = await Promise.all(
+    config.composedPages.map(module => importFromURL(module))
+  );
 
-  // The module pages `all.html` is made of that aren't compiled yet
-  let missingParts;
-
-  /**
-   * Compiles `all.html`, which imports the module pages' compiled programs,
-   * and has the worker pool render it first, as it takes by far the longest.
-   */
-  const writeAllPage = async () => {
-    const allTask = await compile(buildAllPage(compiled));
-
-    rendering.prepend([allTask]);
-
-    allWritten = true;
-  };
+  const composed = createComposedPages(definitions, { compile, rendering });
 
   // With one thread, pages are rendered on this thread once all are compiled,
   // so the manifest is kept until then
@@ -232,13 +222,7 @@ const buildSite = async (input, options) => {
             manifest = item.manifest;
           }
 
-          missingParts = new Set(
-            item.manifest.filter(isModulePage).map(({ api }) => api)
-          );
-
-          for (const { data } of compiled) {
-            missingParts.delete(data.api);
-          }
+          composed.expect(item.manifest);
 
           continue;
         }
@@ -247,17 +231,14 @@ const buildSite = async (input, options) => {
 
         compiled.push(task);
         tasks.push(task);
-
-        missingParts?.delete(task.data.api);
+        composed.add(task);
       }
 
       if (tasks.length > 0) {
         rendering.push(tasks);
       }
 
-      if (!allWritten && missingParts?.size === 0) {
-        await writeAllPage();
-      }
+      await composed.write();
     }
   };
 
@@ -275,9 +256,7 @@ const buildSite = async (input, options) => {
       manifest = undefined;
     }
 
-    if (!allWritten) {
-      await writeAllPage();
-    }
+    await composed.write(true);
 
     htmlLogger.debug(`Compiled ${compiled.length} page programs`);
   } catch (error) {