fix: size the shared crawler pool from the run, not from a request

Author: nikitachapovskii-dev

src/input.ts

@@ -46,6 +46,12 @@ async function getProxyConfiguration(proxyOptions: ProxyOptions) {
 
 const DEFAULT_PROXY_OPTIONS = ragWebBrowserInputSchema.properties.proxyConfiguration.default as ProxyOptions;
 
+/**
+ * How many pages the shared crawlers fetch in parallel to begin with. Set from the run's own input at
+ * startup, never from a request: see `processInput`.
+ */
+let runDesiredConcurrency = ragWebBrowserInputSchema.properties.desiredConcurrency.default;
+
 /**
  * The configuration every crawler is built with. It resolves the proxy from the request Crawlee is
  * about to send rather than from the input the crawler was built with, so one crawler serves callers
@@ -92,6 +98,7 @@ async function validateProxyConfiguration(proxyOptions: ProxyOptions, abortOnFai
 export async function processStandbyInput(originalInput: Partial<Input>) {
     const { input, searchCrawlerOptions, contentScraperSettings } = await processInputInternal(originalInput, true);
 
+    runDesiredConcurrency = input.desiredConcurrency;
     await validateProxyConfiguration(input.proxyConfiguration, true);
     const contentCrawlerOptions: ContentCrawlerOptions[] = [
         createPlaywrightCrawlerOptions(input),
@@ -110,6 +117,13 @@ export async function processInput(originalInput: Partial<Input>) {
     // In Standby this runs once per request, where a proxy problem must fail that request alone. In
     // Normal mode it is the run's startup, where there is nothing to serve and the run ends.
     await validateProxyConfiguration(input.proxyConfiguration, !isActorStandby());
+
+    // A Standby request builds these options whenever no crawler is cached yet, at boot before the
+    // crawlers are up or after a failed one was dropped, so whichever caller got there first would
+    // size the pool that every later caller then shares. The run sizes it instead. Nobody loses a
+    // guarantee: the pool is sized for the instance, not per caller, and it autoscales from here.
+    if (isActorStandby()) input.desiredConcurrency = runDesiredConcurrency;
+
     const contentCrawlerOptions: ContentCrawlerOptions = input.scrapingTool === 'raw-http'
         ? createCheerioCrawlerOptions(input, false)
         : createPlaywrightCrawlerOptions(input, false);

tests/input.test.ts

@@ -1,3 +1,4 @@
+import { Actor } from 'apify';
 import { log } from 'crawlee';
 import { afterEach, describe, expect, it, vi } from 'vitest';
 
@@ -6,6 +7,12 @@ import { parseParameters } from '../src/utils.js';
 
 process.env.ACTOR_FULL_NAME = 'apify/rag-web-browser';
 
+// A custom proxy keeps `Actor.createProxyConfiguration` off the network.
+const CALLER_PROXY = encodeURIComponent(JSON.stringify({
+    useApifyProxy: false,
+    proxyUrls: ['http://caller.invalid:8000'],
+}));
+
 describe('warnIfLowMemoryForPlaywright', () => {
     afterEach(() => {
         vi.restoreAllMocks();
@@ -38,15 +45,28 @@ describe('warnIfLowMemoryForPlaywright', () => {
 
 describe('the log level', () => {
     it('ignores a caller\'s debugMode, which would switch logging for everyone sharing the run', async () => {
-        // A custom proxy keeps `Actor.createProxyConfiguration` off the network.
-        const proxy = encodeURIComponent(JSON.stringify({
-            useApifyProxy: false,
-            proxyUrls: ['http://log-level.invalid:8000'],
-        }));
         log.setLevel(log.LEVELS.INFO);
 
-        await processInput(parseParameters(`?query=hello&debugMode=true&proxyConfiguration=${proxy}`));
+        await processInput(parseParameters(`?query=hello&debugMode=true&proxyConfiguration=${CALLER_PROXY}`));
 
         expect(log.getLevel()).toBe(log.LEVELS.INFO);
     });
 });
+
+describe('the shared crawler pool', () => {
+    afterEach(() => {
+        vi.restoreAllMocks();
+    });
+
+    // A request builds these options whenever no crawler is cached yet, so without this the first
+    // caller through that window would size the pool every later caller shares.
+    it('is sized by the run, not by whichever caller happens to build it', async () => {
+        vi.spyOn(Actor, 'getEnv').mockReturnValue({ metaOrigin: 'STANDBY' } as never);
+
+        const { contentCrawlerOptions } = await processInput(
+            parseParameters(`?query=hello&desiredConcurrency=17&proxyConfiguration=${CALLER_PROXY}`),
+        );
+
+        expect(contentCrawlerOptions.crawlerOptions.autoscaledPoolOptions?.desiredConcurrency).toBe(5);
+    });
+});