From 2ce0fe9a086e00dcaf3aa6f1079de8de72921fe7 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Tue, 3 Nov 2026 19:04:11 +0100 Subject: [PATCH 1/3] perf: let metadata and section-pages start on their dependency's first chunk With more than one thread, metadata and section-pages now start on the first chunk of their dependency instead of waiting for all of it, so parsing, metadata and section pages overlap. The generated files are unchanged. section-pages passes each module's entries on as they arrive and adds the chunk pages after the last of them, in module order, as if built from the whole input at once. A module's chunk pages need all of its entries, and a chunk need not hold whole modules, so groupWholeModules regroups streamed entries into batches of whole modules, and rejects a module whose entries come back after another's. Assisted-by: Claude Opus 5.5 Assisted-by: Claude Sonnet 5.5 --- .changeset/stream-metadata.md | 6 + docs/creating-generators.md | 3 + .../core/src/generators/metadata/index.mjs | 2 + .../core/src/generators/metadata/types.d.ts | 5 +- .../src/utils/__tests__/generators.test.mjs | 106 +++++++++++++++ packages/core/src/utils/generators.mjs | 78 +++++++++++ .../src/html/__tests__/generate.test.mjs | 21 ++- .../section-pages/__tests__/generate.test.mjs | 125 ++++++++++++++++-- packages/react/src/section-pages/generate.mjs | 47 +++++-- packages/react/src/section-pages/index.mjs | 2 + packages/react/src/section-pages/types.d.ts | 5 +- 11 files changed, 378 insertions(+), 22 deletions(-) create mode 100644 .changeset/stream-metadata.md diff --git a/.changeset/stream-metadata.md b/.changeset/stream-metadata.md new file mode 100644 index 000000000..984011c27 --- /dev/null +++ b/.changeset/stream-metadata.md @@ -0,0 +1,6 @@ +--- +'@doc-kit/core': minor +'@doc-kit/generator-react': minor +--- + +`metadata` and `section-pages` stream their input: with more than one thread, they start on their dependency's first chunk, so parsing, metadata and section pages overlap. The generated files are unchanged. `groupWholeModules`, from `@doc-kit/core/utils/generators.mjs`, regroups streamed entries so that each batch holds whole modules. diff --git a/docs/creating-generators.md b/docs/creating-generators.md index 8e15bb89f..d598c1a02 100644 --- a/docs/creating-generators.md +++ b/docs/creating-generators.md @@ -359,6 +359,9 @@ export default { - A dependency that does not stream arrives as a single chunk. - With one thread, `input` is still the collected array, so accept both. - `input` can be iterated only once. +- `metadata`'s chunks hold whole files, but chunks need not hold whole modules: + to get all of a module's entries at once, regroup them with + `groupWholeModules` from `@doc-kit/core/utils/generators.mjs`. ### Non-Streaming Generators diff --git a/packages/core/src/generators/metadata/index.mjs b/packages/core/src/generators/metadata/index.mjs index fcdd965b7..64747e682 100644 --- a/packages/core/src/generators/metadata/index.mjs +++ b/packages/core/src/generators/metadata/index.mjs @@ -16,6 +16,8 @@ export default { hasParallelProcessor: true, + streamsInput: true, + markdown: { remarkPlugins: [ 'remark-parse', diff --git a/packages/core/src/generators/metadata/types.d.ts b/packages/core/src/generators/metadata/types.d.ts index c84305c37..0e04b1e11 100644 --- a/packages/core/src/generators/metadata/types.d.ts +++ b/packages/core/src/generators/metadata/types.d.ts @@ -3,7 +3,10 @@ import type { Position } from 'unist'; export type Generator = GeneratorMetadata< {}, - Generate, AsyncGenerator>, + Generate< + Array | AsyncIterable>, + AsyncGenerator + >, ProcessChunk> >; diff --git a/packages/core/src/utils/__tests__/generators.test.mjs b/packages/core/src/utils/__tests__/generators.test.mjs index 6fe290dba..f81eb1b43 100644 --- a/packages/core/src/utils/__tests__/generators.test.mjs +++ b/packages/core/src/utils/__tests__/generators.test.mjs @@ -4,9 +4,11 @@ import { describe, it } from 'node:test'; import { getEntryDescription, groupNodesByModule, + groupWholeModules, getVersionFromSemVer, coerceSemVer, getCompatibleVersions, + toBatches, } from '../generators.mjs'; describe('getEntryDescription', () => { @@ -86,6 +88,97 @@ describe('groupNodesByModule', () => { }); }); +describe('groupWholeModules', () => { + /** + * Collects the batches regrouped from the given input, each entry as its + * module and number. + * + * @param {Array<{ api: string }> | AsyncIterable>} input + * @returns {Promise} + */ + const collect = async input => { + const output = []; + + for await (const batch of groupWholeModules(input)) { + output.push(batch.map(({ api, n }) => `${api}${n}`)); + } + + return output; + }; + + /** + * Regroups the given batches, streamed in, and collects the batches it + * yields. + * + * @param {Array>} batches + * @returns {Promise} + */ + const regroup = batches => { + async function* input() { + yield* batches; + } + + return collect(input()); + }; + + /** + * An entry of a module, told apart from the module's others by its number. + * + * @param {string} api + * @param {number} n + * @returns {{ api: string, n: number }} + */ + const entry = (api, n) => ({ api, n }); + + it('passes the entries on in the same order, each batch ending once its last module is whole', async () => { + const output = await regroup([ + [entry('a', 1), entry('a', 2), entry('b', 1)], + [entry('c', 1)], + ]); + + assert.deepEqual(output, [['a1', 'a2'], ['b1'], ['c1']]); + }); + + it('holds back a module that continues in the next batch', async () => { + const output = await regroup([ + [entry('a', 1), entry('b', 1)], + [entry('b', 2), entry('b', 3)], + [entry('b', 4), entry('c', 1)], + ]); + + assert.deepEqual(output, [['a1'], ['b1', 'b2', 'b3', 'b4'], ['c1']]); + }); + + it('skips empty batches', async () => { + const output = await regroup([[], [entry('a', 1)], []]); + + assert.deepEqual(output, [['a1']]); + }); + + it('takes entries given as one array as a single batch', async () => { + const output = await collect([entry('a', 1), entry('a', 2), entry('b', 1)]); + + assert.deepEqual(output, [['a1', 'a2'], ['b1']]); + }); + + it('rejects a module whose entries arrive apart', async () => { + await assert.rejects( + regroup([ + [entry('a', 1), entry('b', 1)], + [entry('c', 1), entry('a', 2)], + ]), + /"a" arrived apart/ + ); + }); + + it('rejects a module whose entries arrive apart in one batch too', async () => { + await assert.rejects( + regroup([[entry('a', 1), entry('b', 1), entry('a', 2)]]), + /"a" arrived apart/ + ); + }); +}); + describe('getVersionFromSemVer', () => { it('returns major.x for minor 0', () => { const version = { major: 18, minor: 0, patch: 0 }; @@ -138,3 +231,16 @@ describe('getCompatibleVersions', () => { assert.equal(result.length, 2); }); }); + +describe('toBatches', () => { + it('gives an array as a single batch, and batches as they are', async () => { + async function* batches() { + yield [1]; + } + + const streamed = batches(); + + assert.deepEqual(toBatches([1, 2]), [[1, 2]]); + assert.equal(toBatches(streamed), streamed); + }); +}); diff --git a/packages/core/src/utils/generators.mjs b/packages/core/src/utils/generators.mjs index 516113bca..940eac733 100644 --- a/packages/core/src/utils/generators.mjs +++ b/packages/core/src/utils/generators.mjs @@ -2,6 +2,7 @@ import { coerce, major } from 'semver'; +import { isAsyncIterable } from './misc.mjs'; import { transformNodeToString } from './unist.mjs'; /** @@ -42,6 +43,83 @@ export const groupNodesByModule = nodes => { return Map.groupBy(nodes, node => node.api); }; +/** + * Gives a generator's input as batches: as they arrive if it streams, or as a + * single batch if it arrives whole, as it does with one thread. + * + * @template T + * @param {Array | AsyncIterable>} input + * @returns {Array> | AsyncIterable>} + */ +export const toBatches = input => { + if (isAsyncIterable(input)) { + return input; + } + + return [input]; +}; + +/** + * Regroups streamed entries so that each batch holds whole modules: a module + * whose entries continue in the next batch is held back and passed on with + * it. A module's entries must arrive together: one that comes back after + * another module's is an error, wherever the batches end, as its pages would + * be built without it. Entries given as one array, as a generator that streams + * its input gets them with one thread, are taken as a single batch. + * + * @template {{ api: string }} T + * @param {Array | AsyncIterable>} batches - The entries, in + * batches as they are streamed, or in one array + * @returns {AsyncGenerator>} The same entries, in batches of whole + * modules + */ +export async function* groupWholeModules(batches) { + // The modules whose entries have all arrived + const finished = new Set(); + + // The module whose entries are arriving, and those of them held back + let current; + let held = []; + + for await (const batch of toBatches(batches)) { + for (const { api } of batch) { + if (api !== current) { + if (finished.has(api)) { + throw new Error( + `The entries of "${api}" arrived apart: a module's entries must ` + + 'arrive together, in one chunk or in consecutive ones' + ); + } + + if (current !== undefined) { + finished.add(current); + } + + current = api; + } + } + + const entries = held.length > 0 ? [...held, ...batch] : batch; + + // The module the batch ends with may continue in the next one + let end = entries.length; + + while (end > 0 && entries[end - 1].api === current) { + end--; + } + + held = entries.slice(end); + + if (end > 0) { + yield entries.slice(0, end); + } + } + + if (held.length > 0) { + yield held; + } +} + /** * Parses the SemVer string into a Node.js-alike version * diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs index a2198925b..1e2768215 100644 --- a/packages/react/src/html/__tests__/generate.test.mjs +++ b/packages/react/src/html/__tests__/generate.test.mjs @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { describe, it } from 'node:test'; import { pathToFileURL } from 'node:url'; +import { collectAsyncGenerator } from '@doc-kit/core/caching.mjs'; import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; @@ -12,11 +13,29 @@ import { jsx, toJs } from 'estree-util-to-js'; import buildContent from '../../jsx-ast/utils/buildContent.mjs'; import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs'; -import { generate as chunk } from '../../section-pages/generate.mjs'; +import { generate as sectionPages } from '../../section-pages/generate.mjs'; import { compile, createViteBundler } from '../bundlers/vite.mjs'; import { FONTS } from '../constants.mjs'; import { generate } from '../generate.mjs'; +/** + * Yields the given entries as one batch, as a dependency streaming them does. + * + * @param {Array} entries + */ +async function* streamInOneBatch(entries) { + yield entries; +} + +/** + * Runs `section-pages` over the given entries, streamed in as one batch, and + * collects its pages. + * + * @param {Array} entries + */ +const chunk = entries => + collectAsyncGenerator(sectionPages(streamInOneBatch(entries))); + // Pages are rendered with the pipeline of `jsx-ast` await loadMarkdownPlugins( await loadGenerator(import.meta.resolve('../../jsx-ast/index.mjs')) diff --git a/packages/react/src/section-pages/__tests__/generate.test.mjs b/packages/react/src/section-pages/__tests__/generate.test.mjs index 8605ba1c4..48e02f6e8 100644 --- a/packages/react/src/section-pages/__tests__/generate.test.mjs +++ b/packages/react/src/section-pages/__tests__/generate.test.mjs @@ -7,6 +7,40 @@ import getConfig, { import { generate } from '../generate.mjs'; +/** + * Collects every entry the generator yields. + * + * @param {AsyncIterable>} output + * @returns {Promise>} + */ +const collect = async output => { + const entries = []; + + for await (const batch of output) { + entries.push(...batch); + } + + return entries; +}; + +/** + * Runs the generator over the given batches of entries, streamed in as + * `metadata` yields them, and collects what it yields. + * + * @param {...Array} batches + * @returns {Promise>} + */ +const run = (...batches) => collect(generate(streamOf(batches))); + +/** + * Yields each batch in turn, as a streamed input. + * + * @param {Array>} batches + */ +async function* streamOf(batches) { + yield* batches; +} + /** * Builds a metadata-shaped entry whose heading node is shared between * `heading` and `content`, exactly like the metadata generator produces. @@ -72,7 +106,7 @@ describe('section-pages generate', () => { it('keeps every original entry and appends the chunk pages', async () => { const input = createModule(); - const output = await generate(input); + const output = await run(input); assert.deepEqual(output.slice(0, input.length), input); @@ -97,7 +131,7 @@ describe('section-pages generate', () => { }); it('describes where each chunk came from', async () => { - const output = await generate(createModule()); + const output = await run(createModule()); const readFile = output.find(e => e.path === '/fs/readFile'); assert.deepEqual(readFile.chunk, { @@ -111,7 +145,7 @@ describe('section-pages generate', () => { }); it('promotes headings so the chunk reads as its own page', async () => { - const output = await generate(createModule()); + const output = await run(createModule()); const dir = output.filter(e => e.api === 'fs-Dir'); assert.deepEqual( @@ -134,13 +168,13 @@ describe('section-pages generate', () => { const input = createModule(); const snapshot = structuredClone(input); - await generate(input); + await run(input); assert.deepEqual(input, snapshot); }); it('leaves modules with a single section alone', async () => { - const output = await generate([ + const output = await run([ createEntry('tiny', 1, 'Tiny'), createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), ]); @@ -149,7 +183,7 @@ describe('section-pages generate', () => { }); it('leaves modules that document no API entry alone', async () => { - const output = await generate([ + const output = await run([ createEntry('guide', 1, 'About this documentation'), createEntry('guide', 2, 'Contributing'), createEntry('guide', 2, 'Stability index'), @@ -172,7 +206,7 @@ describe('section-pages generate', () => { link('https://example.com') ); - const output = await generate(input); + const output = await run(input); const chunk = output.find(e => e.path === '/fs/readFile'); assert.deepEqual( @@ -206,7 +240,7 @@ describe('section-pages generate', () => { }, }); - const output = await generate(input); + const output = await run(input); const chunk = output.find(e => e.path === '/fs/readFile'); assert.deepEqual( @@ -225,7 +259,9 @@ describe('section-pages generate', () => { getConfig('section-pages').exclude = ['fs']; try { - assert.equal((await generate(createModule())).length, 6); + const output = await run(createModule()); + + assert.equal(output.length, 6); } finally { getConfig('section-pages').exclude = []; } @@ -235,7 +271,7 @@ describe('section-pages generate', () => { getConfig('section-pages').maxDepth = 2; try { - const output = await generate(createModule()); + const output = await run(createModule()); assert.deepEqual( output.filter(e => e.chunk && e.heading.depth === 1).map(e => e.path), @@ -245,4 +281,73 @@ describe('section-pages generate', () => { getConfig('section-pages').maxDepth = 3; } }); + + it('builds the same output from several batches as from one', async () => { + const tiny = [ + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]; + + const whole = await run([...createModule(), ...tiny]); + const batched = await run(createModule(), tiny); + + assert.deepEqual(batched, whole); + }); + + it('builds the same output from its whole input, as it gets it with one thread', async () => { + const input = () => [ + ...createModule(), + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]; + + const whole = await collect(generate(input())); + const streamed = await run(input()); + + assert.deepEqual(whole, streamed); + }); + + it( + 'passes whole modules on before the next batch arrives', + // The next batch waits for the test, so modules held back until it arrives + // would never be passed on + { timeout: 10_000 }, + async () => { + const gate = Promise.withResolvers(); + const fs = createModule(); + + async function* batches() { + yield [...fs, createEntry('tiny', 1, 'Tiny')]; + await gate.promise; + yield [createEntry('zlib', 1, 'Zlib')]; + } + + const output = generate(batches()); + const { value } = await output.next(); + + // `fs` is whole, `tiny` may continue in the next batch + assert.deepEqual(value, fs); + + gate.resolve(); + + const rest = await collect(output); + + assert.deepEqual( + rest.filter(e => !e.chunk).map(e => e.api), + ['tiny', 'zlib'] + ); + assert.equal( + rest.filter(e => e.chunk && e.heading.depth === 1).length, + 3 + ); + } + ); + + it('builds a module split across batches as one', async () => { + const whole = await run(createModule()); + const module = createModule(); + const split = await run(module.slice(0, 2), module.slice(2)); + + assert.deepEqual(split, whole); + }); }); diff --git a/packages/react/src/section-pages/generate.mjs b/packages/react/src/section-pages/generate.mjs index 12e134cd5..eb50383f7 100644 --- a/packages/react/src/section-pages/generate.mjs +++ b/packages/react/src/section-pages/generate.mjs @@ -1,7 +1,10 @@ 'use strict'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; -import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; +import { + groupNodesByModule, + groupWholeModules, +} from '@doc-kit/core/utils/generators.mjs'; import { visit } from 'unist-util-visit'; import { headingLabel } from '../jsx-ast/utils/buildBarProps.mjs'; @@ -100,14 +103,14 @@ const buildChunkEntries = (module, chunk, index, anchors) => { }; /** - * Adds per-section chunk pages for every module, next to the full pages. + * Builds the chunk pages of every module among the given entries. * - * @type {import('./types').Generator['generate']} + * @param {Array} input - Whole modules' entries + * @param {import('./types').Configuration} config + * @returns {Array} */ -export async function generate(input) { - const { maxDepth, exclude } = getConfig('section-pages'); - - const output = [...input]; +const buildChunkPages = (input, { maxDepth, exclude }) => { + const pages = []; for (const entries of groupNodesByModule(input).values()) { const module = entries.find(entry => entry.heading.depth === 1); @@ -135,9 +138,35 @@ export async function generate(input) { const anchors = buildAnchorMap(chunks); chunks.forEach((chunk, index) => { - output.push(...buildChunkEntries(module, chunk, index, anchors)); + pages.push(...buildChunkEntries(module, chunk, index, anchors)); }); } - return output; + return pages; +}; + +/** + * Adds per-section chunk pages for every module, next to the full pages. + * + * The entries are passed on as their modules arrive, so the pages built from + * them can start while `metadata` still runs, and the chunk pages follow them + * all, in module order, as if built from the whole input at once. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input) { + const config = getConfig('section-pages'); + + const chunkPages = []; + + // A module's chunk pages need all of its entries + for await (const entries of groupWholeModules(input)) { + yield entries; + + chunkPages.push(...buildChunkPages(entries, config)); + } + + if (chunkPages.length > 0) { + yield chunkPages; + } } diff --git a/packages/react/src/section-pages/index.mjs b/packages/react/src/section-pages/index.mjs index b1043d110..380d1883c 100644 --- a/packages/react/src/section-pages/index.mjs +++ b/packages/react/src/section-pages/index.mjs @@ -29,6 +29,8 @@ export default { dependsOn: '@doc-kit/core/metadata', + streamsInput: true, + dependent: [ '@doc-kit/generator-react/html', '@doc-kit/generator-react/sitemap', diff --git a/packages/react/src/section-pages/types.d.ts b/packages/react/src/section-pages/types.d.ts index bf57e1ee2..9d6c579fc 100644 --- a/packages/react/src/section-pages/types.d.ts +++ b/packages/react/src/section-pages/types.d.ts @@ -24,5 +24,8 @@ export type Chunk = { export type Generator = GeneratorMetadata< Configuration, - Generate, Promise>> + Generate< + Array | AsyncIterable>, + AsyncGenerator> + > >; From 2fff56f64cd9e0dba5800298643f8befb736014e Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Wed, 4 Nov 2026 19:07:38 +0100 Subject: [PATCH 2/3] docs: mention toBatches in the changeset, as it comes with groupWholeModules worker.stream no longer reads its input with toBatches, so the helper now comes with groupWholeModules, its first caller. Assisted-by: Claude Opus 5.5 --- .changeset/stream-metadata.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/stream-metadata.md b/.changeset/stream-metadata.md index 984011c27..3cf4f122c 100644 --- a/.changeset/stream-metadata.md +++ b/.changeset/stream-metadata.md @@ -3,4 +3,4 @@ '@doc-kit/generator-react': minor --- -`metadata` and `section-pages` stream their input: with more than one thread, they start on their dependency's first chunk, so parsing, metadata and section pages overlap. The generated files are unchanged. `groupWholeModules`, from `@doc-kit/core/utils/generators.mjs`, regroups streamed entries so that each batch holds whole modules. +`metadata` and `section-pages` stream their input: with more than one thread, they start on their dependency's first chunk, so parsing, metadata and section pages overlap. The generated files are unchanged. `groupWholeModules`, from `@doc-kit/core/utils/generators.mjs`, regroups streamed entries so that each batch holds whole modules, and `toBatches` gives an input that is whole or streamed as batches. From cd9e07804f9c27ef237b4b1f41ce38ce266d37b0 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 9 Nov 2026 12:00:09 +0100 Subject: [PATCH 3/3] refactor: drop groupWholeModules, as metadata's chunks hold whole modules From the review. A module is one file: parseApiDoc sets every entry's api once per file, and worker.stream never splits an item, so each of metadata's chunks already holds whole modules. section-pages reads its input as it comes with toBatches instead of regrouping it, and the helper and its tests go. Co-authored-by: Aviv Keller Assisted-by: Claude Opus 5.5 --- .changeset/stream-metadata.md | 2 +- docs/creating-generators.md | 5 +- .../src/utils/__tests__/generators.test.mjs | 92 ------------------- packages/core/src/utils/generators.mjs | 61 ------------ .../section-pages/__tests__/generate.test.mjs | 23 ++--- packages/react/src/section-pages/generate.mjs | 7 +- 6 files changed, 14 insertions(+), 176 deletions(-) diff --git a/.changeset/stream-metadata.md b/.changeset/stream-metadata.md index 3cf4f122c..6247b9904 100644 --- a/.changeset/stream-metadata.md +++ b/.changeset/stream-metadata.md @@ -3,4 +3,4 @@ '@doc-kit/generator-react': minor --- -`metadata` and `section-pages` stream their input: with more than one thread, they start on their dependency's first chunk, so parsing, metadata and section pages overlap. The generated files are unchanged. `groupWholeModules`, from `@doc-kit/core/utils/generators.mjs`, regroups streamed entries so that each batch holds whole modules, and `toBatches` gives an input that is whole or streamed as batches. +`metadata` and `section-pages` stream their input: with more than one thread, they start on their dependency's first chunk, so parsing, metadata and section pages overlap. The generated files are unchanged. `toBatches`, from `@doc-kit/core/utils/generators.mjs`, gives an input that is whole or streamed as batches. diff --git a/docs/creating-generators.md b/docs/creating-generators.md index d598c1a02..d5bb41adc 100644 --- a/docs/creating-generators.md +++ b/docs/creating-generators.md @@ -359,9 +359,8 @@ export default { - A dependency that does not stream arrives as a single chunk. - With one thread, `input` is still the collected array, so accept both. - `input` can be iterated only once. -- `metadata`'s chunks hold whole files, but chunks need not hold whole modules: - to get all of a module's entries at once, regroup them with - `groupWholeModules` from `@doc-kit/core/utils/generators.mjs`. +- `metadata`'s chunks hold whole files, and a file is one module, so a chunk + holds all the entries of each module in it. ### Non-Streaming Generators diff --git a/packages/core/src/utils/__tests__/generators.test.mjs b/packages/core/src/utils/__tests__/generators.test.mjs index f81eb1b43..a9b550373 100644 --- a/packages/core/src/utils/__tests__/generators.test.mjs +++ b/packages/core/src/utils/__tests__/generators.test.mjs @@ -4,7 +4,6 @@ import { describe, it } from 'node:test'; import { getEntryDescription, groupNodesByModule, - groupWholeModules, getVersionFromSemVer, coerceSemVer, getCompatibleVersions, @@ -88,97 +87,6 @@ describe('groupNodesByModule', () => { }); }); -describe('groupWholeModules', () => { - /** - * Collects the batches regrouped from the given input, each entry as its - * module and number. - * - * @param {Array<{ api: string }> | AsyncIterable>} input - * @returns {Promise} - */ - const collect = async input => { - const output = []; - - for await (const batch of groupWholeModules(input)) { - output.push(batch.map(({ api, n }) => `${api}${n}`)); - } - - return output; - }; - - /** - * Regroups the given batches, streamed in, and collects the batches it - * yields. - * - * @param {Array>} batches - * @returns {Promise} - */ - const regroup = batches => { - async function* input() { - yield* batches; - } - - return collect(input()); - }; - - /** - * An entry of a module, told apart from the module's others by its number. - * - * @param {string} api - * @param {number} n - * @returns {{ api: string, n: number }} - */ - const entry = (api, n) => ({ api, n }); - - it('passes the entries on in the same order, each batch ending once its last module is whole', async () => { - const output = await regroup([ - [entry('a', 1), entry('a', 2), entry('b', 1)], - [entry('c', 1)], - ]); - - assert.deepEqual(output, [['a1', 'a2'], ['b1'], ['c1']]); - }); - - it('holds back a module that continues in the next batch', async () => { - const output = await regroup([ - [entry('a', 1), entry('b', 1)], - [entry('b', 2), entry('b', 3)], - [entry('b', 4), entry('c', 1)], - ]); - - assert.deepEqual(output, [['a1'], ['b1', 'b2', 'b3', 'b4'], ['c1']]); - }); - - it('skips empty batches', async () => { - const output = await regroup([[], [entry('a', 1)], []]); - - assert.deepEqual(output, [['a1']]); - }); - - it('takes entries given as one array as a single batch', async () => { - const output = await collect([entry('a', 1), entry('a', 2), entry('b', 1)]); - - assert.deepEqual(output, [['a1', 'a2'], ['b1']]); - }); - - it('rejects a module whose entries arrive apart', async () => { - await assert.rejects( - regroup([ - [entry('a', 1), entry('b', 1)], - [entry('c', 1), entry('a', 2)], - ]), - /"a" arrived apart/ - ); - }); - - it('rejects a module whose entries arrive apart in one batch too', async () => { - await assert.rejects( - regroup([[entry('a', 1), entry('b', 1), entry('a', 2)]]), - /"a" arrived apart/ - ); - }); -}); - describe('getVersionFromSemVer', () => { it('returns major.x for minor 0', () => { const version = { major: 18, minor: 0, patch: 0 }; diff --git a/packages/core/src/utils/generators.mjs b/packages/core/src/utils/generators.mjs index 940eac733..f8996d2b5 100644 --- a/packages/core/src/utils/generators.mjs +++ b/packages/core/src/utils/generators.mjs @@ -59,67 +59,6 @@ export const toBatches = input => { return [input]; }; -/** - * Regroups streamed entries so that each batch holds whole modules: a module - * whose entries continue in the next batch is held back and passed on with - * it. A module's entries must arrive together: one that comes back after - * another module's is an error, wherever the batches end, as its pages would - * be built without it. Entries given as one array, as a generator that streams - * its input gets them with one thread, are taken as a single batch. - * - * @template {{ api: string }} T - * @param {Array | AsyncIterable>} batches - The entries, in - * batches as they are streamed, or in one array - * @returns {AsyncGenerator>} The same entries, in batches of whole - * modules - */ -export async function* groupWholeModules(batches) { - // The modules whose entries have all arrived - const finished = new Set(); - - // The module whose entries are arriving, and those of them held back - let current; - let held = []; - - for await (const batch of toBatches(batches)) { - for (const { api } of batch) { - if (api !== current) { - if (finished.has(api)) { - throw new Error( - `The entries of "${api}" arrived apart: a module's entries must ` + - 'arrive together, in one chunk or in consecutive ones' - ); - } - - if (current !== undefined) { - finished.add(current); - } - - current = api; - } - } - - const entries = held.length > 0 ? [...held, ...batch] : batch; - - // The module the batch ends with may continue in the next one - let end = entries.length; - - while (end > 0 && entries[end - 1].api === current) { - end--; - } - - held = entries.slice(end); - - if (end > 0) { - yield entries.slice(0, end); - } - } - - if (held.length > 0) { - yield held; - } -} - /** * Parses the SemVer string into a Node.js-alike version * diff --git a/packages/react/src/section-pages/__tests__/generate.test.mjs b/packages/react/src/section-pages/__tests__/generate.test.mjs index 48e02f6e8..2e5f07d59 100644 --- a/packages/react/src/section-pages/__tests__/generate.test.mjs +++ b/packages/react/src/section-pages/__tests__/generate.test.mjs @@ -308,16 +308,16 @@ describe('section-pages generate', () => { }); it( - 'passes whole modules on before the next batch arrives', - // The next batch waits for the test, so modules held back until it arrives - // would never be passed on + 'passes each batch on before the next one arrives', + // The next batch waits for the test, so a batch held back until it + // arrives would never be passed on { timeout: 10_000 }, async () => { const gate = Promise.withResolvers(); - const fs = createModule(); + const first = [...createModule(), createEntry('tiny', 1, 'Tiny')]; async function* batches() { - yield [...fs, createEntry('tiny', 1, 'Tiny')]; + yield first; await gate.promise; yield [createEntry('zlib', 1, 'Zlib')]; } @@ -325,8 +325,7 @@ describe('section-pages generate', () => { const output = generate(batches()); const { value } = await output.next(); - // `fs` is whole, `tiny` may continue in the next batch - assert.deepEqual(value, fs); + assert.deepEqual(value, first); gate.resolve(); @@ -334,7 +333,7 @@ describe('section-pages generate', () => { assert.deepEqual( rest.filter(e => !e.chunk).map(e => e.api), - ['tiny', 'zlib'] + ['zlib'] ); assert.equal( rest.filter(e => e.chunk && e.heading.depth === 1).length, @@ -342,12 +341,4 @@ describe('section-pages generate', () => { ); } ); - - it('builds a module split across batches as one', async () => { - const whole = await run(createModule()); - const module = createModule(); - const split = await run(module.slice(0, 2), module.slice(2)); - - assert.deepEqual(split, whole); - }); }); diff --git a/packages/react/src/section-pages/generate.mjs b/packages/react/src/section-pages/generate.mjs index eb50383f7..13450139a 100644 --- a/packages/react/src/section-pages/generate.mjs +++ b/packages/react/src/section-pages/generate.mjs @@ -3,7 +3,7 @@ import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; import { groupNodesByModule, - groupWholeModules, + toBatches, } from '@doc-kit/core/utils/generators.mjs'; import { visit } from 'unist-util-visit'; @@ -159,8 +159,9 @@ export async function* generate(input) { const chunkPages = []; - // A module's chunk pages need all of its entries - for await (const entries of groupWholeModules(input)) { + // A module's chunk pages need all of its entries, which arrive together: + // each of `metadata`'s chunks holds whole files, and a file is one module + for await (const entries of toBatches(input)) { yield entries; chunkPages.push(...buildChunkPages(entries, config));