perf: keep none of the CLI's target output

.changeset/drain-results.md

@@ -0,0 +1,6 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/cli': patch
+---
+
+`runGenerators(configuration, { results: false })` keeps none of the requested generators' output: each still runs to its end, and once nothing else reads its output, its chunks go as they are recorded. The CLI's `generate` and `preview` commands use it, as they don't read the results.

packages/cli/bin/commands/generate.mjs

@@ -64,6 +64,6 @@ export default generate.action(
     const config = await setConfig(opts);
     assertRunnableOptions(config);
 
-    await runGenerators(config);
+    await runGenerators(config, { results: false });
   })
 );

packages/cli/bin/commands/preview.mjs

@@ -62,7 +62,7 @@ export default preview.action(
       const config = await setConfig(opts);
       assertRunnableOptions(config);
 
-      await runGenerators(config);
+      await runGenerators(config, { results: false });
 
       // With the default coming from the configuration, this is always set
       output = config.global.output;

packages/core/src/__tests__/caching.test.mjs

@@ -188,5 +188,28 @@ describe('caching', () => {
 
       deepStrictEqual(chunks, [[1]]);
     });
+
+    it('should drain a result, alongside its other readers', async () => {
+      const cache = createCache();
+
+      cache.populateConsumerCounts(['a', 'b'], name =>
+        name === 'b' ? 'a' : undefined
+      );
+
+      cache.store(
+        'a',
+        (async function* () {
+          yield [1];
+          yield [2];
+        })()
+      );
+
+      const reader = await cache.stream('a');
+
+      await cache.drain('a');
+
+      strictEqual(cache.has('a'), false);
+      deepStrictEqual(await collectChunks(reader), [[1], [2]]);
+    });
   });
 });

packages/core/src/__tests__/generators.test.mjs

@@ -123,6 +123,14 @@ const syntheticGenerators = {
       }
     },
   },
+  failing: {
+    name: 'failing',
+    generate: async () =>
+      (async function* () {
+        yield [1];
+        throw new Error('source failed');
+      })(),
+  },
   'shape-reader': {
     name: 'shape-reader',
     dependsOn: 'metadata',
@@ -292,4 +300,27 @@ describe('createGenerator orchestration', () => {
     assert.deepStrictEqual(await run(1), [true]);
     assert.deepStrictEqual(await run(2), [false]);
   });
+
+  it('runs every generator but keeps no results with results: false', async () => {
+    const { runGenerators } = createGenerator();
+
+    const results = await runGenerators(
+      { target: ['gen-c', 'gen-c-all'], threads: 1 },
+      { results: false }
+    );
+
+    assert.equal(results, undefined);
+    assert.equal(runs['gen-c'], 1);
+    assert.equal(runs['gen-c-all'], 1);
+  });
+
+  it('rejects with a failure with results: false', async () => {
+    await assert.rejects(
+      createGenerator().runGenerators(
+        { target: ['failing'], threads: 1 },
+        { results: false }
+      ),
+      /source failed/
+    );
+  });
 });

packages/core/src/caching.mjs

@@ -210,6 +210,32 @@ export const createCache = () => {
       return value;
     },
 
+    /**
+     * Runs a generator to its end, keeping none of its result, which counts as
+     * one of its reads.
+     *
+     * @param {string} name - Cache key (resolved generator specifier) to drain
+     * @returns {Promise<void>}
+     */
+    drain: async name => {
+      const result = await results[name];
+
+      let reader = [];
+
+      if (isAsyncIterable(result)) {
+        // The last read of a result nothing else has read needn't share it
+        const last = remaining[name] === 1 && !shared.has(name);
+
+        reader = last ? result : read(name, result);
+      }
+
+      release(name);
+
+      for await (const chunk of reader) {
+        void chunk;
+      }
+    },
+
     /**
      * Reads a generator's result chunk by chunk as it is produced (a result
      * that doesn't stream is a single chunk), counting as one consumer.

packages/core/src/generators.mjs

@@ -112,9 +112,13 @@ const createGenerator = () => {
    * Runs all requested generators with their dependencies.
    *
    * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options
-   * @returns {Promise<unknown[]>} Results of all requested generators
+   * @param {{ results?: boolean }} [options] - With `results: false`, none of
+   * the requested generators' results is kept, so each can be let go as soon
+   * as it is produced
+   * @returns {Promise<unknown[] | undefined>} Results of all requested
+   * generators, or `undefined` with `results: false`
    */
-  const runGenerators = async configuration => {
+  const runGenerators = async (configuration, { results = true } = {}) => {
     const { target, threads } = configuration;
 
     // Resolve shorthand names and load the full dependency closure up front,
@@ -147,15 +151,14 @@ const createGenerator = () => {
       scheduleGenerator(specifier, generators, inputOf, configuration);
     }
 
-    // Start all collections in parallel (don't await sequentially). Consuming
-    // through the shared path lets the final read also trigger eviction.
-    const results = await Promise.all(
-      targets.map(specifier => cache.consume(specifier))
-    );
+    // Start all reads in parallel (don't await sequentially). Reading through
+    // the shared path lets the final read also trigger eviction.
+    const read = results ? cache.consume : cache.drain;
+    const collected = await Promise.all(targets.map(read));
 
     await pool.destroy();
 
-    return results;
+    return results ? collected : undefined;
   };
 
   return { runGenerators };