feat: implement wait for dynamic content timeout

Author: Patai5

code/src/configuration.ts

@@ -16,6 +16,7 @@ const ajv = new Ajv.default();
  */
 export const parseConfiguration = async (input: Input): Promise<Config> => {
     const {
+        dynamicContentWaitSecs = 0,
         excludeUrlGlobs,
         includeUrlGlobs,
         initialCookies,
@@ -51,6 +52,7 @@ export const parseConfiguration = async (input: Input): Promise<Config> => {
     const maxCrawlingDepth = input.maxCrawlingDepth || Number.POSITIVE_INFINITY;
 
     return {
+        dynamicContentWaitSecs,
         excludeUrlGlobs,
         includeUrlGlobs,
         initialCookies,

code/src/routes/crawl-route.ts

@@ -1,5 +1,7 @@
 import { Actor } from 'apify';
-import { Dataset, KeyValueStore, NonRetryableError, PlaywrightCrawlingContext, log, utils } from 'crawlee';
+import { Dataset, KeyValueStore, NonRetryableError, PlaywrightCrawlingContext, log, sleep, utils } from 'crawlee';
+import { Page } from 'playwright';
+
 import { validateInputCssSelectors } from '../configuration.js';
 import { ERROR_OCCURRED_MESSAGE, NonRetryableOpenaiAPIError, OpenaiAPIErrorToExitActor } from '../errors.js';
 import { getNumberOfTextTokens, htmlToMarkdown, maybeShortsTextByTokenLength, shrinkHtml } from '../processors.js';
@@ -17,6 +19,7 @@ export const crawlRoute = async (context: PlaywrightCrawlingContext) => {
 
     const state = await crawler.useState<CrawlerState>();
     const {
+        dynamicContentWaitSecs,
         excludeUrlGlobs,
         includeUrlGlobs,
         instructions,
@@ -64,6 +67,7 @@ export const crawlRoute = async (context: PlaywrightCrawlingContext) => {
 
     log.info(`Opening ${url}...`);
 
+    await waitForDynamicContent(page, dynamicContentWaitSecs);
     await closeCookieModals();
 
     // Enqueue links
@@ -209,3 +213,14 @@ export const crawlRoute = async (context: PlaywrightCrawlingContext) => {
         },
     });
 };
+
+/**
+ * Waits for dynamic content to load on the page.
+ * - Waits for the given `timeoutS` to pass, but breaks early if the network is idle (loaded all resources).
+ */
+const waitForDynamicContent = async (page: Page, timeoutS: number) => {
+    const networkIdlePromise = page.waitForLoadState('networkidle');
+    const timeoutPromise = sleep(timeoutS * 1000);
+
+    return Promise.race([networkIdlePromise, timeoutPromise]);
+};

code/src/types/config.ts

@@ -8,6 +8,7 @@ import { OpenAIModelSettings } from './models.js';
  * Parsed input configuration.
  */
 export interface Config {
+    dynamicContentWaitSecs: number;
     excludeUrlGlobs?: GlobInput[];
     includeUrlGlobs?: GlobInput[];
     initialCookies?: Cookie[];

code/src/types/input.ts

@@ -8,6 +8,7 @@ import { ValuesOf } from './utils.js';
  * Input schema in TypeScript format.
  */
 export type Input = (OpenAIModelSettings & DeprecatedInput) & {
+    dynamicContentWaitSecs?: number;
     startUrls: RequestOptions[];
     includeUrlGlobs?: GlobInput[];
     excludeUrlGlobs?: GlobInput[];