From 60077dd0134ac77183aaf3837cc0eb251c3947d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Pr=C5=AF=C5=A1a?= Date: Tue, 30 Jul 2024 18:25:35 +0200 Subject: [PATCH] fix: set scale down factor to be faster --- code/src/crawler.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/code/src/crawler.ts b/code/src/crawler.ts index b59b001..3533b6e 100644 --- a/code/src/crawler.ts +++ b/code/src/crawler.ts @@ -15,8 +15,11 @@ export const createCrawler = async (config: Config) => { headless: true, }, }, - /** The default values scale up and down too quickly for larger runs, these values are half that */ - autoscaledPoolOptions: { scaleDownStepRatio: 0.025, scaleUpStepRatio: 0.025 }, + /** + * The default value scale up too quickly for larger runs, this value is half that + * - Scaling down is still the default value, meaning the pool will scale down faster than it scales up + */ + autoscaledPoolOptions: { scaleUpStepRatio: 0.025 }, retryOnBlocked: true, requestHandlerTimeoutSecs: 3 * 60, proxyConfiguration,