perf: let metadata and section-pages start on their dependency's first chunk

.changeset/stream-metadata.md

@@ -0,0 +1,6 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+---
+
+`metadata` and `section-pages` stream their input: with more than one thread, they start on their dependency's first chunk, so parsing, metadata and section pages overlap. The generated files are unchanged. `toBatches`, from `@doc-kit/core/utils/generators.mjs`, gives an input that is whole or streamed as batches.

docs/creating-generators.md

@@ -359,6 +359,8 @@ export default {
 - A dependency that does not stream arrives as a single chunk.
 - With one thread, `input` is still the collected array, so accept both.
 - `input` can be iterated only once.
+- `metadata`'s chunks hold whole files, and a file is one module, so a chunk
+  holds all the entries of each module in it.
 
 ### Non-Streaming Generators
 

packages/core/src/generators/metadata/index.mjs

@@ -16,6 +16,8 @@ export default {
 
   hasParallelProcessor: true,
 
+  streamsInput: true,
+
   markdown: {
     remarkPlugins: [
       'remark-parse',

packages/core/src/generators/metadata/types.d.ts

@@ -3,7 +3,10 @@ import type { Position } from 'unist';
 
 export type Generator = GeneratorMetadata<
   {},
-  Generate<Array<Root>, AsyncGenerator<MetadataEntry>>,
+  Generate<
+    Array<Root> | AsyncIterable<Array<Root>>,
+    AsyncGenerator<MetadataEntry>
+  >,
   ProcessChunk<Root, MetadataEntry, Record<string, string>>
 >;
 

packages/core/src/utils/__tests__/generators.test.mjs

@@ -7,6 +7,7 @@ import {
   getVersionFromSemVer,
   coerceSemVer,
   getCompatibleVersions,
+  toBatches,
 } from '../generators.mjs';
 
 describe('getEntryDescription', () => {
@@ -138,3 +139,16 @@ describe('getCompatibleVersions', () => {
     assert.equal(result.length, 2);
   });
 });
+
+describe('toBatches', () => {
+  it('gives an array as a single batch, and batches as they are', async () => {
+    async function* batches() {
+      yield [1];
+    }
+
+    const streamed = batches();
+
+    assert.deepEqual(toBatches([1, 2]), [[1, 2]]);
+    assert.equal(toBatches(streamed), streamed);
+  });
+});

packages/core/src/utils/generators.mjs

@@ -2,6 +2,7 @@
 
 import { coerce, major } from 'semver';
 
+import { isAsyncIterable } from './misc.mjs';
 import { transformNodeToString } from './unist.mjs';
 
 /**
@@ -42,6 +43,22 @@ export const groupNodesByModule = nodes => {
   return Map.groupBy(nodes, node => node.api);
 };
 
+/**
+ * Gives a generator's input as batches: as they arrive if it streams, or as a
+ * single batch if it arrives whole, as it does with one thread.
+ *
+ * @template T
+ * @param {Array<T> | AsyncIterable<Array<T>>} input
+ * @returns {Array<Array<T>> | AsyncIterable<Array<T>>}
+ */
+export const toBatches = input => {
+  if (isAsyncIterable(input)) {
+    return input;
+  }
+
+  return [input];
+};
+
 /**
  * Parses the SemVer string into a Node.js-alike version
  *

packages/react/src/html/__tests__/generate.test.mjs

@@ -5,18 +5,37 @@ import { join } from 'node:path';
 import { describe, it } from 'node:test';
 import { pathToFileURL } from 'node:url';
 
+import { collectAsyncGenerator } from '@doc-kit/core/caching.mjs';
 import { loadGenerator } from '@doc-kit/core/generators/loader.mjs';
 import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs';
 import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
 import { jsx, toJs } from 'estree-util-to-js';
 
 import buildContent from '../../jsx-ast/utils/buildContent.mjs';
 import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs';
-import { generate as chunk } from '../../section-pages/generate.mjs';
+import { generate as sectionPages } from '../../section-pages/generate.mjs';
 import { compile, createViteBundler } from '../bundlers/vite.mjs';
 import { FONTS } from '../constants.mjs';
 import { generate } from '../generate.mjs';
 
+/**
+ * Yields the given entries as one batch, as a dependency streaming them does.
+ *
+ * @param {Array<object>} entries
+ */
+async function* streamInOneBatch(entries) {
+  yield entries;
+}
+
+/**
+ * Runs `section-pages` over the given entries, streamed in as one batch, and
+ * collects its pages.
+ *
+ * @param {Array<object>} entries
+ */
+const chunk = entries =>
+  collectAsyncGenerator(sectionPages(streamInOneBatch(entries)));
+
 // Pages are rendered with the pipeline of `jsx-ast`
 await loadMarkdownPlugins(
   await loadGenerator(import.meta.resolve('../../jsx-ast/index.mjs'))

packages/react/src/section-pages/__tests__/generate.test.mjs

@@ -7,6 +7,40 @@ import getConfig, {
 
 import { generate } from '../generate.mjs';
 
+/**
+ * Collects every entry the generator yields.
+ *
+ * @param {AsyncIterable<Array<object>>} output
+ * @returns {Promise<Array<object>>}
+ */
+const collect = async output => {
+  const entries = [];
+
+  for await (const batch of output) {
+    entries.push(...batch);
+  }
+
+  return entries;
+};
+
+/**
+ * Runs the generator over the given batches of entries, streamed in as
+ * `metadata` yields them, and collects what it yields.
+ *
+ * @param {...Array<object>} batches
+ * @returns {Promise<Array<object>>}
+ */
+const run = (...batches) => collect(generate(streamOf(batches)));
+
+/**
+ * Yields each batch in turn, as a streamed input.
+ *
+ * @param {Array<Array<object>>} batches
+ */
+async function* streamOf(batches) {
+  yield* batches;
+}
+
 /**
  * Builds a metadata-shaped entry whose heading node is shared between
  * `heading` and `content`, exactly like the metadata generator produces.
@@ -72,7 +106,7 @@ describe('section-pages generate', () => {
 
   it('keeps every original entry and appends the chunk pages', async () => {
     const input = createModule();
-    const output = await generate(input);
+    const output = await run(input);
 
     assert.deepEqual(output.slice(0, input.length), input);
 
@@ -97,7 +131,7 @@ describe('section-pages generate', () => {
   });
 
   it('describes where each chunk came from', async () => {
-    const output = await generate(createModule());
+    const output = await run(createModule());
     const readFile = output.find(e => e.path === '/fs/readFile');
 
     assert.deepEqual(readFile.chunk, {
@@ -111,7 +145,7 @@ describe('section-pages generate', () => {
   });
 
   it('promotes headings so the chunk reads as its own page', async () => {
-    const output = await generate(createModule());
+    const output = await run(createModule());
 
     const dir = output.filter(e => e.api === 'fs-Dir');
     assert.deepEqual(
@@ -134,13 +168,13 @@ describe('section-pages generate', () => {
     const input = createModule();
     const snapshot = structuredClone(input);
 
-    await generate(input);
+    await run(input);
 
     assert.deepEqual(input, snapshot);
   });
 
   it('leaves modules with a single section alone', async () => {
-    const output = await generate([
+    const output = await run([
       createEntry('tiny', 1, 'Tiny'),
       createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }),
     ]);
@@ -149,7 +183,7 @@ describe('section-pages generate', () => {
   });
 
   it('leaves modules that document no API entry alone', async () => {
-    const output = await generate([
+    const output = await run([
       createEntry('guide', 1, 'About this documentation'),
       createEntry('guide', 2, 'Contributing'),
       createEntry('guide', 2, 'Stability index'),
@@ -172,7 +206,7 @@ describe('section-pages generate', () => {
       link('https://example.com')
     );
 
-    const output = await generate(input);
+    const output = await run(input);
     const chunk = output.find(e => e.path === '/fs/readFile');
 
     assert.deepEqual(
@@ -206,7 +240,7 @@ describe('section-pages generate', () => {
       },
     });
 
-    const output = await generate(input);
+    const output = await run(input);
     const chunk = output.find(e => e.path === '/fs/readFile');
 
     assert.deepEqual(
@@ -225,7 +259,9 @@ describe('section-pages generate', () => {
     getConfig('section-pages').exclude = ['fs'];
 
     try {
-      assert.equal((await generate(createModule())).length, 6);
+      const output = await run(createModule());
+
+      assert.equal(output.length, 6);
     } finally {
       getConfig('section-pages').exclude = [];
     }
@@ -235,7 +271,7 @@ describe('section-pages generate', () => {
     getConfig('section-pages').maxDepth = 2;
 
     try {
-      const output = await generate(createModule());
+      const output = await run(createModule());
 
       assert.deepEqual(
         output.filter(e => e.chunk && e.heading.depth === 1).map(e => e.path),
@@ -245,4 +281,64 @@ describe('section-pages generate', () => {
       getConfig('section-pages').maxDepth = 3;
     }
   });
+
+  it('builds the same output from several batches as from one', async () => {
+    const tiny = [
+      createEntry('tiny', 1, 'Tiny'),
+      createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }),
+    ];
+
+    const whole = await run([...createModule(), ...tiny]);
+    const batched = await run(createModule(), tiny);
+
+    assert.deepEqual(batched, whole);
+  });
+
+  it('builds the same output from its whole input, as it gets it with one thread', async () => {
+    const input = () => [
+      ...createModule(),
+      createEntry('tiny', 1, 'Tiny'),
+      createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }),
+    ];
+
+    const whole = await collect(generate(input()));
+    const streamed = await run(input());
+
+    assert.deepEqual(whole, streamed);
+  });
+
+  it(
+    'passes each batch on before the next one arrives',
+    // The next batch waits for the test, so a batch held back until it
+    // arrives would never be passed on
+    { timeout: 10_000 },
+    async () => {
+      const gate = Promise.withResolvers();
+      const first = [...createModule(), createEntry('tiny', 1, 'Tiny')];
+
+      async function* batches() {
+        yield first;
+        await gate.promise;
+        yield [createEntry('zlib', 1, 'Zlib')];
+      }
+
+      const output = generate(batches());
+      const { value } = await output.next();
+
+      assert.deepEqual(value, first);
+
+      gate.resolve();
+
+      const rest = await collect(output);
+
+      assert.deepEqual(
+        rest.filter(e => !e.chunk).map(e => e.api),
+        ['zlib']
+      );
+      assert.equal(
+        rest.filter(e => e.chunk && e.heading.depth === 1).length,
+        3
+      );
+    }
+  );
 });

packages/react/src/section-pages/generate.mjs

@@ -1,7 +1,10 @@
 'use strict';
 
 import getConfig from '@doc-kit/core/utils/configuration/index.mjs';
-import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs';
+import {
+  groupNodesByModule,
+  toBatches,
+} from '@doc-kit/core/utils/generators.mjs';
 import { visit } from 'unist-util-visit';
 
 import { headingLabel } from '../jsx-ast/utils/buildBarProps.mjs';
@@ -100,14 +103,14 @@ const buildChunkEntries = (module, chunk, index, anchors) => {
 };
 
 /**
- * Adds per-section chunk pages for every module, next to the full pages.
+ * Builds the chunk pages of every module among the given entries.
  *
- * @type {import('./types').Generator['generate']}
+ * @param {Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>} input - Whole modules' entries
+ * @param {import('./types').Configuration} config
+ * @returns {Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>}
  */
-export async function generate(input) {
-  const { maxDepth, exclude } = getConfig('section-pages');
-
-  const output = [...input];
+const buildChunkPages = (input, { maxDepth, exclude }) => {
+  const pages = [];
 
   for (const entries of groupNodesByModule(input).values()) {
     const module = entries.find(entry => entry.heading.depth === 1);
@@ -135,9 +138,36 @@ export async function generate(input) {
     const anchors = buildAnchorMap(chunks);
 
     chunks.forEach((chunk, index) => {
-      output.push(...buildChunkEntries(module, chunk, index, anchors));
+      pages.push(...buildChunkEntries(module, chunk, index, anchors));
     });
   }
 
-  return output;
+  return pages;
+};
+
+/**
+ * Adds per-section chunk pages for every module, next to the full pages.
+ *
+ * The entries are passed on as their modules arrive, so the pages built from
+ * them can start while `metadata` still runs, and the chunk pages follow them
+ * all, in module order, as if built from the whole input at once.
+ *
+ * @type {import('./types').Generator['generate']}
+ */
+export async function* generate(input) {
+  const config = getConfig('section-pages');
+
+  const chunkPages = [];
+
+  // A module's chunk pages need all of its entries, which arrive together:
+  // each of `metadata`'s chunks holds whole files, and a file is one module
+  for await (const entries of toBatches(input)) {
+    yield entries;
+
+    chunkPages.push(...buildChunkPages(entries, config));
+  }
+
+  if (chunkPages.length > 0) {
+    yield chunkPages;
+  }
 }