diff --git a/.changeset/stream-metadata.md b/.changeset/stream-metadata.md new file mode 100644 index 000000000..6247b9904 --- /dev/null +++ b/.changeset/stream-metadata.md @@ -0,0 +1,6 @@ +--- +'@doc-kit/core': minor +'@doc-kit/generator-react': minor +--- + +`metadata` and `section-pages` stream their input: with more than one thread, they start on their dependency's first chunk, so parsing, metadata and section pages overlap. The generated files are unchanged. `toBatches`, from `@doc-kit/core/utils/generators.mjs`, gives an input that is whole or streamed as batches. diff --git a/docs/creating-generators.md b/docs/creating-generators.md index 8e15bb89f..d5bb41adc 100644 --- a/docs/creating-generators.md +++ b/docs/creating-generators.md @@ -359,6 +359,8 @@ export default { - A dependency that does not stream arrives as a single chunk. - With one thread, `input` is still the collected array, so accept both. - `input` can be iterated only once. +- `metadata`'s chunks hold whole files, and a file is one module, so a chunk + holds all the entries of each module in it. ### Non-Streaming Generators diff --git a/packages/core/src/generators/metadata/index.mjs b/packages/core/src/generators/metadata/index.mjs index fcdd965b7..64747e682 100644 --- a/packages/core/src/generators/metadata/index.mjs +++ b/packages/core/src/generators/metadata/index.mjs @@ -16,6 +16,8 @@ export default { hasParallelProcessor: true, + streamsInput: true, + markdown: { remarkPlugins: [ 'remark-parse', diff --git a/packages/core/src/generators/metadata/types.d.ts b/packages/core/src/generators/metadata/types.d.ts index c84305c37..0e04b1e11 100644 --- a/packages/core/src/generators/metadata/types.d.ts +++ b/packages/core/src/generators/metadata/types.d.ts @@ -3,7 +3,10 @@ import type { Position } from 'unist'; export type Generator = GeneratorMetadata< {}, - Generate, AsyncGenerator>, + Generate< + Array | AsyncIterable>, + AsyncGenerator + >, ProcessChunk> >; diff --git a/packages/core/src/utils/__tests__/generators.test.mjs b/packages/core/src/utils/__tests__/generators.test.mjs index 6fe290dba..a9b550373 100644 --- a/packages/core/src/utils/__tests__/generators.test.mjs +++ b/packages/core/src/utils/__tests__/generators.test.mjs @@ -7,6 +7,7 @@ import { getVersionFromSemVer, coerceSemVer, getCompatibleVersions, + toBatches, } from '../generators.mjs'; describe('getEntryDescription', () => { @@ -138,3 +139,16 @@ describe('getCompatibleVersions', () => { assert.equal(result.length, 2); }); }); + +describe('toBatches', () => { + it('gives an array as a single batch, and batches as they are', async () => { + async function* batches() { + yield [1]; + } + + const streamed = batches(); + + assert.deepEqual(toBatches([1, 2]), [[1, 2]]); + assert.equal(toBatches(streamed), streamed); + }); +}); diff --git a/packages/core/src/utils/generators.mjs b/packages/core/src/utils/generators.mjs index 516113bca..f8996d2b5 100644 --- a/packages/core/src/utils/generators.mjs +++ b/packages/core/src/utils/generators.mjs @@ -2,6 +2,7 @@ import { coerce, major } from 'semver'; +import { isAsyncIterable } from './misc.mjs'; import { transformNodeToString } from './unist.mjs'; /** @@ -42,6 +43,22 @@ export const groupNodesByModule = nodes => { return Map.groupBy(nodes, node => node.api); }; +/** + * Gives a generator's input as batches: as they arrive if it streams, or as a + * single batch if it arrives whole, as it does with one thread. + * + * @template T + * @param {Array | AsyncIterable>} input + * @returns {Array> | AsyncIterable>} + */ +export const toBatches = input => { + if (isAsyncIterable(input)) { + return input; + } + + return [input]; +}; + /** * Parses the SemVer string into a Node.js-alike version * diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs index a2198925b..1e2768215 100644 --- a/packages/react/src/html/__tests__/generate.test.mjs +++ b/packages/react/src/html/__tests__/generate.test.mjs @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { describe, it } from 'node:test'; import { pathToFileURL } from 'node:url'; +import { collectAsyncGenerator } from '@doc-kit/core/caching.mjs'; import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; @@ -12,11 +13,29 @@ import { jsx, toJs } from 'estree-util-to-js'; import buildContent from '../../jsx-ast/utils/buildContent.mjs'; import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs'; -import { generate as chunk } from '../../section-pages/generate.mjs'; +import { generate as sectionPages } from '../../section-pages/generate.mjs'; import { compile, createViteBundler } from '../bundlers/vite.mjs'; import { FONTS } from '../constants.mjs'; import { generate } from '../generate.mjs'; +/** + * Yields the given entries as one batch, as a dependency streaming them does. + * + * @param {Array} entries + */ +async function* streamInOneBatch(entries) { + yield entries; +} + +/** + * Runs `section-pages` over the given entries, streamed in as one batch, and + * collects its pages. + * + * @param {Array} entries + */ +const chunk = entries => + collectAsyncGenerator(sectionPages(streamInOneBatch(entries))); + // Pages are rendered with the pipeline of `jsx-ast` await loadMarkdownPlugins( await loadGenerator(import.meta.resolve('../../jsx-ast/index.mjs')) diff --git a/packages/react/src/section-pages/__tests__/generate.test.mjs b/packages/react/src/section-pages/__tests__/generate.test.mjs index 8605ba1c4..2e5f07d59 100644 --- a/packages/react/src/section-pages/__tests__/generate.test.mjs +++ b/packages/react/src/section-pages/__tests__/generate.test.mjs @@ -7,6 +7,40 @@ import getConfig, { import { generate } from '../generate.mjs'; +/** + * Collects every entry the generator yields. + * + * @param {AsyncIterable>} output + * @returns {Promise>} + */ +const collect = async output => { + const entries = []; + + for await (const batch of output) { + entries.push(...batch); + } + + return entries; +}; + +/** + * Runs the generator over the given batches of entries, streamed in as + * `metadata` yields them, and collects what it yields. + * + * @param {...Array} batches + * @returns {Promise>} + */ +const run = (...batches) => collect(generate(streamOf(batches))); + +/** + * Yields each batch in turn, as a streamed input. + * + * @param {Array>} batches + */ +async function* streamOf(batches) { + yield* batches; +} + /** * Builds a metadata-shaped entry whose heading node is shared between * `heading` and `content`, exactly like the metadata generator produces. @@ -72,7 +106,7 @@ describe('section-pages generate', () => { it('keeps every original entry and appends the chunk pages', async () => { const input = createModule(); - const output = await generate(input); + const output = await run(input); assert.deepEqual(output.slice(0, input.length), input); @@ -97,7 +131,7 @@ describe('section-pages generate', () => { }); it('describes where each chunk came from', async () => { - const output = await generate(createModule()); + const output = await run(createModule()); const readFile = output.find(e => e.path === '/fs/readFile'); assert.deepEqual(readFile.chunk, { @@ -111,7 +145,7 @@ describe('section-pages generate', () => { }); it('promotes headings so the chunk reads as its own page', async () => { - const output = await generate(createModule()); + const output = await run(createModule()); const dir = output.filter(e => e.api === 'fs-Dir'); assert.deepEqual( @@ -134,13 +168,13 @@ describe('section-pages generate', () => { const input = createModule(); const snapshot = structuredClone(input); - await generate(input); + await run(input); assert.deepEqual(input, snapshot); }); it('leaves modules with a single section alone', async () => { - const output = await generate([ + const output = await run([ createEntry('tiny', 1, 'Tiny'), createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), ]); @@ -149,7 +183,7 @@ describe('section-pages generate', () => { }); it('leaves modules that document no API entry alone', async () => { - const output = await generate([ + const output = await run([ createEntry('guide', 1, 'About this documentation'), createEntry('guide', 2, 'Contributing'), createEntry('guide', 2, 'Stability index'), @@ -172,7 +206,7 @@ describe('section-pages generate', () => { link('https://example.com') ); - const output = await generate(input); + const output = await run(input); const chunk = output.find(e => e.path === '/fs/readFile'); assert.deepEqual( @@ -206,7 +240,7 @@ describe('section-pages generate', () => { }, }); - const output = await generate(input); + const output = await run(input); const chunk = output.find(e => e.path === '/fs/readFile'); assert.deepEqual( @@ -225,7 +259,9 @@ describe('section-pages generate', () => { getConfig('section-pages').exclude = ['fs']; try { - assert.equal((await generate(createModule())).length, 6); + const output = await run(createModule()); + + assert.equal(output.length, 6); } finally { getConfig('section-pages').exclude = []; } @@ -235,7 +271,7 @@ describe('section-pages generate', () => { getConfig('section-pages').maxDepth = 2; try { - const output = await generate(createModule()); + const output = await run(createModule()); assert.deepEqual( output.filter(e => e.chunk && e.heading.depth === 1).map(e => e.path), @@ -245,4 +281,64 @@ describe('section-pages generate', () => { getConfig('section-pages').maxDepth = 3; } }); + + it('builds the same output from several batches as from one', async () => { + const tiny = [ + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]; + + const whole = await run([...createModule(), ...tiny]); + const batched = await run(createModule(), tiny); + + assert.deepEqual(batched, whole); + }); + + it('builds the same output from its whole input, as it gets it with one thread', async () => { + const input = () => [ + ...createModule(), + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]; + + const whole = await collect(generate(input())); + const streamed = await run(input()); + + assert.deepEqual(whole, streamed); + }); + + it( + 'passes each batch on before the next one arrives', + // The next batch waits for the test, so a batch held back until it + // arrives would never be passed on + { timeout: 10_000 }, + async () => { + const gate = Promise.withResolvers(); + const first = [...createModule(), createEntry('tiny', 1, 'Tiny')]; + + async function* batches() { + yield first; + await gate.promise; + yield [createEntry('zlib', 1, 'Zlib')]; + } + + const output = generate(batches()); + const { value } = await output.next(); + + assert.deepEqual(value, first); + + gate.resolve(); + + const rest = await collect(output); + + assert.deepEqual( + rest.filter(e => !e.chunk).map(e => e.api), + ['zlib'] + ); + assert.equal( + rest.filter(e => e.chunk && e.heading.depth === 1).length, + 3 + ); + } + ); }); diff --git a/packages/react/src/section-pages/generate.mjs b/packages/react/src/section-pages/generate.mjs index 12e134cd5..13450139a 100644 --- a/packages/react/src/section-pages/generate.mjs +++ b/packages/react/src/section-pages/generate.mjs @@ -1,7 +1,10 @@ 'use strict'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; -import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; +import { + groupNodesByModule, + toBatches, +} from '@doc-kit/core/utils/generators.mjs'; import { visit } from 'unist-util-visit'; import { headingLabel } from '../jsx-ast/utils/buildBarProps.mjs'; @@ -100,14 +103,14 @@ const buildChunkEntries = (module, chunk, index, anchors) => { }; /** - * Adds per-section chunk pages for every module, next to the full pages. + * Builds the chunk pages of every module among the given entries. * - * @type {import('./types').Generator['generate']} + * @param {Array} input - Whole modules' entries + * @param {import('./types').Configuration} config + * @returns {Array} */ -export async function generate(input) { - const { maxDepth, exclude } = getConfig('section-pages'); - - const output = [...input]; +const buildChunkPages = (input, { maxDepth, exclude }) => { + const pages = []; for (const entries of groupNodesByModule(input).values()) { const module = entries.find(entry => entry.heading.depth === 1); @@ -135,9 +138,36 @@ export async function generate(input) { const anchors = buildAnchorMap(chunks); chunks.forEach((chunk, index) => { - output.push(...buildChunkEntries(module, chunk, index, anchors)); + pages.push(...buildChunkEntries(module, chunk, index, anchors)); }); } - return output; + return pages; +}; + +/** + * Adds per-section chunk pages for every module, next to the full pages. + * + * The entries are passed on as their modules arrive, so the pages built from + * them can start while `metadata` still runs, and the chunk pages follow them + * all, in module order, as if built from the whole input at once. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input) { + const config = getConfig('section-pages'); + + const chunkPages = []; + + // A module's chunk pages need all of its entries, which arrive together: + // each of `metadata`'s chunks holds whole files, and a file is one module + for await (const entries of toBatches(input)) { + yield entries; + + chunkPages.push(...buildChunkPages(entries, config)); + } + + if (chunkPages.length > 0) { + yield chunkPages; + } } diff --git a/packages/react/src/section-pages/index.mjs b/packages/react/src/section-pages/index.mjs index b1043d110..380d1883c 100644 --- a/packages/react/src/section-pages/index.mjs +++ b/packages/react/src/section-pages/index.mjs @@ -29,6 +29,8 @@ export default { dependsOn: '@doc-kit/core/metadata', + streamsInput: true, + dependent: [ '@doc-kit/generator-react/html', '@doc-kit/generator-react/sitemap', diff --git a/packages/react/src/section-pages/types.d.ts b/packages/react/src/section-pages/types.d.ts index bf57e1ee2..9d6c579fc 100644 --- a/packages/react/src/section-pages/types.d.ts +++ b/packages/react/src/section-pages/types.d.ts @@ -24,5 +24,8 @@ export type Chunk = { export type Generator = GeneratorMetadata< Configuration, - Generate, Promise>> + Generate< + Array | AsyncIterable>, + AsyncGenerator> + > >;