diff --git a/.changeset/drain-results.md b/.changeset/drain-results.md new file mode 100644 index 000000000..0170fe5dd --- /dev/null +++ b/.changeset/drain-results.md @@ -0,0 +1,6 @@ +--- +'@doc-kit/core': minor +'@doc-kit/cli': patch +--- + +`runGenerators(configuration, { results: false })` keeps none of the requested generators' output: each still runs to its end, and once nothing else reads its output, its chunks go as they are recorded. The CLI's `generate` and `preview` commands use it, as they don't read the results. diff --git a/packages/cli/bin/commands/generate.mjs b/packages/cli/bin/commands/generate.mjs index 4122f6fb3..358fcf039 100644 --- a/packages/cli/bin/commands/generate.mjs +++ b/packages/cli/bin/commands/generate.mjs @@ -64,6 +64,6 @@ export default generate.action( const config = await setConfig(opts); assertRunnableOptions(config); - await runGenerators(config); + await runGenerators(config, { results: false }); }) ); diff --git a/packages/cli/bin/commands/preview.mjs b/packages/cli/bin/commands/preview.mjs index 84272fe67..e8b8e4d56 100644 --- a/packages/cli/bin/commands/preview.mjs +++ b/packages/cli/bin/commands/preview.mjs @@ -62,7 +62,7 @@ export default preview.action( const config = await setConfig(opts); assertRunnableOptions(config); - await runGenerators(config); + await runGenerators(config, { results: false }); // With the default coming from the configuration, this is always set output = config.global.output; diff --git a/packages/core/src/__tests__/caching.test.mjs b/packages/core/src/__tests__/caching.test.mjs index 47e942fa2..122c64748 100644 --- a/packages/core/src/__tests__/caching.test.mjs +++ b/packages/core/src/__tests__/caching.test.mjs @@ -188,5 +188,28 @@ describe('caching', () => { deepStrictEqual(chunks, [[1]]); }); + + it('should drain a result, alongside its other readers', async () => { + const cache = createCache(); + + cache.populateConsumerCounts(['a', 'b'], name => + name === 'b' ? 'a' : undefined + ); + + cache.store( + 'a', + (async function* () { + yield [1]; + yield [2]; + })() + ); + + const reader = await cache.stream('a'); + + await cache.drain('a'); + + strictEqual(cache.has('a'), false); + deepStrictEqual(await collectChunks(reader), [[1], [2]]); + }); }); }); diff --git a/packages/core/src/__tests__/generators.test.mjs b/packages/core/src/__tests__/generators.test.mjs index 0eafdb3cc..746e49cb0 100644 --- a/packages/core/src/__tests__/generators.test.mjs +++ b/packages/core/src/__tests__/generators.test.mjs @@ -123,6 +123,14 @@ const syntheticGenerators = { } }, }, + failing: { + name: 'failing', + generate: async () => + (async function* () { + yield [1]; + throw new Error('source failed'); + })(), + }, 'shape-reader': { name: 'shape-reader', dependsOn: 'metadata', @@ -292,4 +300,27 @@ describe('createGenerator orchestration', () => { assert.deepStrictEqual(await run(1), [true]); assert.deepStrictEqual(await run(2), [false]); }); + + it('runs every generator but keeps no results with results: false', async () => { + const { runGenerators } = createGenerator(); + + const results = await runGenerators( + { target: ['gen-c', 'gen-c-all'], threads: 1 }, + { results: false } + ); + + assert.equal(results, undefined); + assert.equal(runs['gen-c'], 1); + assert.equal(runs['gen-c-all'], 1); + }); + + it('rejects with a failure with results: false', async () => { + await assert.rejects( + createGenerator().runGenerators( + { target: ['failing'], threads: 1 }, + { results: false } + ), + /source failed/ + ); + }); }); diff --git a/packages/core/src/caching.mjs b/packages/core/src/caching.mjs index 99118dc0c..c7da15a37 100644 --- a/packages/core/src/caching.mjs +++ b/packages/core/src/caching.mjs @@ -210,6 +210,32 @@ export const createCache = () => { return value; }, + /** + * Runs a generator to its end, keeping none of its result, which counts as + * one of its reads. + * + * @param {string} name - Cache key (resolved generator specifier) to drain + * @returns {Promise} + */ + drain: async name => { + const result = await results[name]; + + let reader = []; + + if (isAsyncIterable(result)) { + // The last read of a result nothing else has read needn't share it + const last = remaining[name] === 1 && !shared.has(name); + + reader = last ? result : read(name, result); + } + + release(name); + + for await (const chunk of reader) { + void chunk; + } + }, + /** * Reads a generator's result chunk by chunk as it is produced (a result * that doesn't stream is a single chunk), counting as one consumer. diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index 97159d0cf..53604a59d 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -112,9 +112,13 @@ const createGenerator = () => { * Runs all requested generators with their dependencies. * * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options - * @returns {Promise} Results of all requested generators + * @param {{ results?: boolean }} [options] - With `results: false`, none of + * the requested generators' results is kept, so each can be let go as soon + * as it is produced + * @returns {Promise} Results of all requested + * generators, or `undefined` with `results: false` */ - const runGenerators = async configuration => { + const runGenerators = async (configuration, { results = true } = {}) => { const { target, threads } = configuration; // Resolve shorthand names and load the full dependency closure up front, @@ -147,15 +151,14 @@ const createGenerator = () => { scheduleGenerator(specifier, generators, inputOf, configuration); } - // Start all collections in parallel (don't await sequentially). Consuming - // through the shared path lets the final read also trigger eviction. - const results = await Promise.all( - targets.map(specifier => cache.consume(specifier)) - ); + // Start all reads in parallel (don't await sequentially). Reading through + // the shared path lets the final read also trigger eviction. + const read = results ? cache.consume : cache.drain; + const collected = await Promise.all(targets.map(read)); await pool.destroy(); - return results; + return results ? collected : undefined; }; return { runGenerators };