fix: serve every standby request from the crawlers started at boot

Author: nikitachapovskii-dev

src/const.ts

@@ -18,6 +18,17 @@ export enum ContentCrawlerTypes {
     CHEERIO = 'cheerio',
 }
 
+/** Identifies which of the crawlers a key belongs to. */
+export type CrawlerKind = 'search' | ContentCrawlerTypes;
+
+/**
+ * The crawlers are shared by every Standby request, so these cannot be built from one request's input.
+ * They are the widest the input schemas allow; each request then narrows them down to its own values,
+ * `maxRequestRetries` through `Request.maxRetries` and the timeout through a pre-navigation hook.
+ */
+export const CRAWLER_MAX_REQUEST_RETRIES = 5;
+export const CRAWLER_REQUEST_HANDLER_TIMEOUT_SECS = 300;
+
 export const PLAYWRIGHT_REQUEST_TIMEOUT_NORMAL_MODE_SECS = 60;
 
 export const GOOGLE_STANDARD_RESULTS_PER_PAGE = 10;

src/crawlers.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 
 import { ImpitHttpClient } from '@crawlee/impit-client';
@@ -18,6 +19,7 @@ import {
 } from 'crawlee';
 
 import { chargeFetch, chargeSearch } from './charging.js';
+import type { CrawlerKind } from './const.js';
 import { ContentCrawlerTypes, GOOGLE_STANDARD_RESULTS_PER_PAGE } from './const.js';
 import { deduplicateResults, scrapeOrganicResults } from './google-search/google-extractors-urls.js';
 import {
@@ -27,10 +29,19 @@ import {
     TEXT_DOCUMENT_CONTENT_TYPES,
 } from './request-handler.js';
 import { addEmptyResultToResponse, addResultToResponse, sendResponseError, sendResponseIfFinished } from './responses.js';
-import type { ContentCrawlerOptions, ContentCrawlerUserData, Output, SearchCrawlerUserData } from './types.js';
+import type {
+    ContentCrawlerOptions,
+    ContentCrawlerUserData,
+    Output,
+    ProxyOptions,
+    SearchCrawlerOptions,
+    SearchCrawlerUserData,
+} from './types.js';
 import { addTimeMeasureEvent, createRequest, createSearchRequest } from './utils.js';
 
-const crawlers = new Map<string, CheerioCrawler | PlaywrightCrawler>();
+type Crawler = CheerioCrawler | PlaywrightCrawler;
+
+const crawlers = new Map<string, Crawler>();
 const client = new MemoryStorage({ persistStorage: false });
 
 const contentCrawlerHttpClient = new ImpitHttpClient({
@@ -56,26 +67,58 @@ async function getGhosteryBlocker(): Promise<PlaywrightBlocker | undefined> {
     }
 }
 
-export function getCrawlerKey(crawlerOptions: CheerioCrawlerOptions | PlaywrightCrawlerOptions) {
-    return JSON.stringify(crawlerOptions);
+/**
+ * Identifies a crawler in the `crawlers` cache, and names its request queue. Keyed off the caller's
+ * proxy options rather than the built `ProxyConfiguration`, whose child logger snapshots the log level
+ * and whose URL rotation state mutates - both of which kept changing the key. Hashed because the key
+ * is logged and used as a queue name, while the options can hold credentials.
+ */
+export function getCrawlerKey(kind: CrawlerKind, proxyOptions: ProxyOptions): string {
+    const hash = createHash('sha1').update(JSON.stringify(proxyOptions)).digest('hex');
+    return `${kind}-${hash.slice(0, 12)}`;
+}
+
+/** How many crawlers are alive. Exported so tests can assert that requests do not fork new ones. */
+export function getCrawlerCount() {
+    return crawlers.size;
+}
+
+async function getOrCreateCrawler(key: string, kind: CrawlerKind, startCrawler: boolean, build: () => Promise<Crawler>) {
+    const cached = crawlers.get(key);
+    if (cached) {
+        return cached;
+    }
+
+    log.info(`Creating new ${kind} crawler with key ${key}`);
+    const crawler = await build();
+    if (startCrawler) {
+        crawler.run().then(
+            () => log.warning(`Crawler ${kind} has finished`),
+            // eslint-disable-next-line @typescript-eslint/no-empty-function
+            () => { },
+        );
+        log.info(`Crawler ${kind} has started 💪🏼`);
+    }
+    crawlers.set(key, crawler);
+    log.info(`Number of crawlers ${crawlers.size}`);
+    return crawler;
 }
 
 /**
- * Adds a content crawl request to selected content crawler.
- * Get existing crawler based on crawlerOptions and scraperSettings, if not present -> create new
+ * Adds a content crawl request to the content crawler identified by `contentCrawlerKey`.
  */
 export const addContentCrawlRequest = async (
     request: RequestOptions<ContentCrawlerUserData>,
     responseId: string,
     contentCrawlerKey: string,
 ) => {
     const crawler = crawlers.get(contentCrawlerKey);
-    const name = crawler instanceof PlaywrightCrawler ? 'playwright' : 'cheerio';
-
     if (!crawler) {
-        log.error(`Content crawler not found: key ${contentCrawlerKey}`);
+        log.error(`Content crawler not found: ${contentCrawlerKey}`);
         return;
     }
+
+    const name = crawler instanceof PlaywrightCrawler ? 'playwright' : 'cheerio';
     try {
         await crawler.requestQueue!.addRequest(request);
         // create an empty result in search request response
@@ -92,17 +135,13 @@ export const addContentCrawlRequest = async (
  * A crawler won't be created if it already exists.
  */
 export async function createAndStartSearchCrawler(
-    searchCrawlerOptions: CheerioCrawlerOptions,
+    searchCrawlerOptions: SearchCrawlerOptions,
     startCrawler = true,
 ) {
-    const key = getCrawlerKey(searchCrawlerOptions);
-    if (crawlers.has(key)) {
-        return { key, crawler: crawlers.get(key) };
-    }
-
-    log.info(`Creating new cheerio crawler with key ${key}`);
-    const crawler = new CheerioCrawler({
-        ...(searchCrawlerOptions as CheerioCrawlerOptions),
+    const { crawlerOptions, proxyOptions } = searchCrawlerOptions;
+    const key = getCrawlerKey('search', proxyOptions);
+    const crawler = await getOrCreateCrawler(key, 'search', startCrawler, async () => new CheerioCrawler({
+        ...crawlerOptions,
         requestQueue: await RequestQueue.open(key, { storageClient: client }),
         requestHandler: async ({ request, $: _$, addRequests }: CheerioCrawlingContext<SearchCrawlerUserData>) => {
             // NOTE: we need to cast this to fix `cheerio` type errors
@@ -150,7 +189,7 @@ export async function createAndStartSearchCrawler(
                         collectedResults: deduplicated,
                         currentPage: nextPage,
                     },
-                    searchCrawlerOptions.proxyConfiguration,
+                    proxyOptions,
                     nextOffset,
                 );
                 await addRequests([nextRequest]);
@@ -182,17 +221,8 @@ export async function createAndStartSearchCrawler(
             const errorResponse = { errorMessage: err.message };
             sendResponseError(request.uniqueKey, JSON.stringify(errorResponse));
         },
-    });
-    if (startCrawler) {
-        crawler.run().then(
-            () => log.warning('Google-search-crawler has finished'),
-            // eslint-disable-next-line @typescript-eslint/no-empty-function
-            () => { },
-        );
-        log.info('Google-search-crawler has started 🫡');
-    }
-    crawlers.set(key, crawler);
-    log.info(`Number of crawlers ${crawlers.size}`);
+    }));
+
     return { key, crawler };
 }
 
@@ -205,27 +235,15 @@ export async function createAndStartContentCrawler(
     contentCrawlerOptions: ContentCrawlerOptions,
     startCrawler = true,
 ) {
-    const { type: crawlerType, crawlerOptions } = contentCrawlerOptions;
+    const { type: crawlerType, crawlerOptions, proxyOptions } = contentCrawlerOptions;
 
-    const key = getCrawlerKey(crawlerOptions);
-    if (crawlers.has(key)) {
-        return { key, crawler: crawlers.get(key) };
-    }
+    const key = getCrawlerKey(crawlerType, proxyOptions);
+    const crawler = await getOrCreateCrawler(key, crawlerType, startCrawler, async () => (
+        crawlerType === ContentCrawlerTypes.PLAYWRIGHT
+            ? createPlaywrightContentCrawler(crawlerOptions, key)
+            : createCheerioContentCrawler(crawlerOptions, key)
+    ));
 
-    const crawler = crawlerType === 'playwright'
-        ? await createPlaywrightContentCrawler(crawlerOptions, key)
-        : await createCheerioContentCrawler(crawlerOptions, key);
-
-    if (startCrawler) {
-        crawler.run().then(
-            () => log.warning(`Crawler ${crawlerType} has finished`),
-            // eslint-disable-next-line @typescript-eslint/no-empty-function
-            () => { },
-        );
-        log.info(`Crawler ${crawlerType} has started 💪🏼`);
-    }
-    crawlers.set(key, crawler);
-    log.info(`Number of crawlers ${crawlers.size}`);
     return { key, crawler };
 }
 
@@ -261,7 +279,6 @@ async function createPlaywrightContentCrawler(
     crawlerOptions: PlaywrightCrawlerOptions,
     key: string,
 ): Promise<PlaywrightCrawler> {
-    log.info(`Creating new playwright crawler with key ${key}`);
     const blocker = await getGhosteryBlocker();
     return new PlaywrightCrawler({
         ...crawlerOptions,
@@ -284,7 +301,6 @@ async function createCheerioContentCrawler(
     crawlerOptions: CheerioCrawlerOptions,
     key: string,
 ): Promise<CheerioCrawler> {
-    log.info(`Creating new cheerio crawler with key ${key}`);
     return new CheerioCrawler({
         ...crawlerOptions,
         keepAlive: crawlerOptions.keepAlive,
@@ -305,20 +321,19 @@ async function createCheerioContentCrawler(
 }
 
 /**
- * Adds a search request to the Google search crawler.
+ * Adds a search request to the search crawler identified by `searchCrawlerKey`.
  * Create a response for the request and set the desired number of results (maxResults).
  */
 export const addSearchRequest = async (
     request: RequestOptions<ContentCrawlerUserData>,
-    searchCrawlerOptions: CheerioCrawlerOptions,
+    searchCrawlerKey: string,
 ) => {
-    const key = getCrawlerKey(searchCrawlerOptions);
-    const crawler = crawlers.get(key);
-
+    const crawler = crawlers.get(searchCrawlerKey);
     if (!crawler) {
-        log.error(`Cheerio crawler not found: key ${key}`);
+        log.error(`Search crawler not found: ${searchCrawlerKey}`);
         return;
     }
+
     addTimeMeasureEvent(request.userData!, 'before-cheerio-queue-add');
     await crawler.requestQueue!.addRequest(request);
     log.info(`Added request to cheerio-google-search-crawler: ${request.url}`);