fix: remove key value stores from config

apify-projects · Feb 18, 2024 · 3af94de · 3af94de
1 parent 684425e
commit 3af94de
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 32 deletions.
diff --git a/packages/gpt-scraper-core/src/configuration.ts b/packages/gpt-scraper-core/src/configuration.ts
@@ -1,14 +1,14 @@
 import { AnySchema } from 'ajv';
 import addFormats from 'ajv-formats';
-import Ajv2020 from 'ajv/dist/2020';
+import Ajv2020 from 'ajv/dist/2020.js';
 import { Actor } from 'apify';
-import { Cookie, KeyValueStore, RequestList, log } from 'crawlee';
+import { Cookie, RequestList, log } from 'crawlee';
 import { Page } from 'playwright';
-import { getModelByName } from './models/models';
-import { OpenAIModelHandler } from './models/openai';
+import { getModelByName } from './models/models.js';
+import { OpenAIModelHandler } from './models/openai.js';
 import { OpenAIModelSettings } from './types';
-import { Config } from './types/config';
-import { Input, PAGE_FORMAT } from './types/input';
+import { Config } from './types/config.js';
+import { Input, PAGE_FORMAT } from './types/input.js';
 
 /**
  * Parses the Actor's input into a config object and validates it. Throws an Actor fail if the input is invalid.
@@ -39,8 +39,7 @@ export const parseConfiguration = async (input: Input): Promise<Config> => {
 
     const proxyConfiguration = await Actor.createProxyConfiguration(input.proxyConfiguration);
 
-    const requestList = await RequestList.open('start-urls', startUrls);
-    const kvStore = await KeyValueStore.open();
+    const { requests } = await RequestList.open({ sources: startUrls });
 
     // make sure to change 0 (unlimited) to a very high number, because this is used in arithmetics and comparisons
     const maxCrawlingDepth = input.maxCrawlingDepth || 999999;
@@ -51,7 +50,7 @@ export const parseConfiguration = async (input: Input): Promise<Config> => {
         includeUrlGlobs,
         initialCookies,
         instructions,
-        kvStore,
+        requests,
         linkSelector,
         maxCrawlingDepth,
         maxPagesPerCrawl,
@@ -60,7 +59,6 @@ export const parseConfiguration = async (input: Input): Promise<Config> => {
         pageFormat: pageFormatInRequest,
         proxyConfiguration,
         removeElementsCssSelector,
-        requestList,
         saveSnapshots,
         schema,
         skipGptGlobs,

diff --git a/packages/gpt-scraper-core/src/crawler.ts b/packages/gpt-scraper-core/src/crawler.ts
@@ -9,7 +9,7 @@ import { ERROR_TYPE } from './utils.js';
 export const createCrawler = async ({ input }: { input: Input }) => {
     const config = await parseConfiguration(input);
 
-    const { maxPagesPerCrawl, requestList, proxyConfiguration } = config;
+    const { maxPagesPerCrawl, proxyConfiguration, requests } = config;
 
     const crawler = new PlaywrightCrawler({
         launchContext: {
@@ -22,7 +22,6 @@ export const createCrawler = async ({ input }: { input: Input }) => {
         requestHandlerTimeoutSecs: 3 * 60,
         proxyConfiguration,
         maxRequestsPerCrawl: maxPagesPerCrawl,
-        requestList,
         requestHandler: crawlRoute,
         preNavigationHooks: [
             async () => {
@@ -69,6 +68,8 @@ export const createCrawler = async ({ input }: { input: Input }) => {
     const defaultCrawlerState = { pagesOpened: 0, config };
     await crawler.useState<CrawlerState>(defaultCrawlerState);
 
+    await crawler.addRequests(requests);
+
     // @ts-expect-error patching
     const oldCrawlerLogError = crawler.log.error.bind(crawler.log);
     // @ts-expect-error patching

diff --git a/packages/gpt-scraper-core/src/routes/crawl-route.ts b/packages/gpt-scraper-core/src/routes/crawl-route.ts
@@ -1,35 +1,36 @@
 import { Actor } from 'apify';
-import { Dataset, NonRetryableError, PlaywrightCrawlingContext, log, utils } from 'crawlee';
-import { validateInputCssSelectors } from '../configuration';
-import { ERROR_OCCURRED_MESSAGE, NonRetryableOpenaiAPIError, OpenaiAPIErrorToExitActor } from '../errors';
-import { getNumberOfTextTokens, htmlToMarkdown, maybeShortsTextByTokenLength, shrinkHtml } from '../processors';
-import { PAGE_FORMAT } from '../types';
-import { CrawlerState } from '../types/crawler-state';
-import { ERROR_TYPE, doesUrlMatchGlobs } from '../utils';
+import { Dataset, KeyValueStore, NonRetryableError, PlaywrightCrawlingContext, log, utils } from 'crawlee';
+import { validateInputCssSelectors } from '../configuration.js';
+import { ERROR_OCCURRED_MESSAGE, NonRetryableOpenaiAPIError, OpenaiAPIErrorToExitActor } from '../errors.js';
+import { getNumberOfTextTokens, htmlToMarkdown, maybeShortsTextByTokenLength, shrinkHtml } from '../processors.js';
+import { CrawlerState } from '../types/crawler-state.js';
+import { PAGE_FORMAT } from '../types/input.js';
+import { ERROR_TYPE, doesUrlMatchGlobs } from '../utils.js';
 
 /**
  * The main crawling route. Enqueues new URLs and processes the page by calling the GPT model.
  */
 export const crawlRoute = async (context: PlaywrightCrawlingContext) => {
     const { request, page, enqueueLinks, closeCookieModals, crawler } = context;
 
+    const kvStore = await KeyValueStore.open();
+
     const state = await crawler.useState<CrawlerState>();
     const {
-        maxPagesPerCrawl,
-        maxCrawlingDepth,
-        includeUrlGlobs,
-        linkSelector,
         excludeUrlGlobs,
-        skipGptGlobs,
-        targetSelector,
-        removeElementsCssSelector,
-        pageFormat,
+        includeUrlGlobs,
         instructions,
+        linkSelector,
+        maxCrawlingDepth,
+        maxPagesPerCrawl,
         model,
+        modelSettings,
+        pageFormat,
+        removeElementsCssSelector,
         saveSnapshots,
-        kvStore,
         schema,
-        modelSettings,
+        skipGptGlobs,
+        targetSelector,
     } = state.config;
 
     const { depth = 0 } = request.userData;

diff --git a/packages/gpt-scraper-core/src/types/config.ts b/packages/gpt-scraper-core/src/types/config.ts
@@ -1,5 +1,5 @@
 import { AnySchema } from 'ajv';
-import { Cookie, GlobInput, KeyValueStore, ProxyConfiguration, RequestList } from 'crawlee';
+import { Cookie, GlobInput, ProxyConfiguration, Request } from 'crawlee';
 import { OpenAIModelHandler } from '../models/openai';
 import { PAGE_FORMAT } from './input';
 import { OpenAIModelSettings } from './models';
@@ -12,7 +12,6 @@ export interface Config {
     includeUrlGlobs?: GlobInput[];
     initialCookies?: Cookie[];
     instructions: string;
-    kvStore: KeyValueStore;
     linkSelector?: string;
     maxCrawlingDepth: number;
     maxPagesPerCrawl: number;
@@ -21,7 +20,7 @@ export interface Config {
     pageFormat: PAGE_FORMAT;
     proxyConfiguration?: ProxyConfiguration;
     removeElementsCssSelector?: string;
-    requestList: RequestList;
+    requests: Request[];
     saveSnapshots: boolean;
     schema?: AnySchema;
     skipGptGlobs?: GlobInput[];