Skip to content

Commit bd4062e

Browse files
dandakaclaude
andcommitted
feat: add Bun runtime detection and CI testing
Add runtime detection utilities (`isBunRuntime`, `warnIfBunRuntime`) to `@crawlee/utils` and wire a one-time warning into `GotScrapingHttpClient` so users running under Bun get a clear message suggesting `ImpitHttpClient` instead of a cryptic got-scraping failure. Add a `bun_test` CI job that runs core, storage, session-pool, router, and HTTP crawler tests under Bun to catch regressions. Relates to #2046 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 9b93877 commit bd4062e

5 files changed

Lines changed: 156 additions & 1 deletion

File tree

.github/workflows/test-ci.yml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,69 @@ jobs:
100100
env:
101101
YARN_IGNORE_NODE: 1
102102

103+
bun_test:
104+
name: Bun Compatibility
105+
if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
106+
runs-on: ubuntu-22.04
107+
108+
steps:
109+
- name: Checkout repository
110+
uses: actions/checkout@v7
111+
112+
- name: Use Node.js 24
113+
uses: actions/setup-node@v6
114+
with:
115+
node-version: 24
116+
package-manager-cache: false
117+
118+
- name: Setup Bun
119+
uses: oven-sh/setup-bun@v2
120+
with:
121+
bun-version: latest
122+
123+
- name: Enable corepack
124+
run: |
125+
corepack enable
126+
corepack prepare yarn@stable --activate
127+
128+
- name: Activate cache for Node.js 24
129+
uses: actions/setup-node@v6
130+
with:
131+
cache: 'yarn'
132+
133+
- name: Turbo cache
134+
id: turbo-cache
135+
uses: actions/cache@v5
136+
with:
137+
path: .turbo
138+
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
139+
restore-keys: |
140+
turbo-${{ github.job }}-${{ github.ref_name }}-
141+
142+
- name: Install Dependencies
143+
run: yarn
144+
env:
145+
YARN_IGNORE_NODE: 1
146+
147+
- name: Build
148+
run: yarn ci:build
149+
env:
150+
YARN_IGNORE_NODE: 1
151+
152+
- name: Bun compatibility tests
153+
run: |
154+
bun run vitest run \
155+
test/core/runtime.test.ts \
156+
test/core/proxy_configuration.test.ts \
157+
test/core/base_http_client.test.ts \
158+
test/core/crawlers/http_crawler.test.ts \
159+
test/core/storages/ \
160+
test/core/session_pool/ \
161+
test/core/router.test.ts \
162+
test/core/error_tracker.test.ts
163+
env:
164+
YARN_IGNORE_NODE: 1
165+
103166
docs:
104167
name: Docs build
105168
if: (!contains(github.event.head_commit.message, '[skip ci]') && github.ref != 'refs/heads/master')

packages/core/src/http_clients/got-scraping-http-client.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { gotScraping } from '@crawlee/utils';
1+
import { gotScraping, warnIfBunRuntime } from '@crawlee/utils';
22
// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood
33
import type { Options, PlainResponse } from 'got-scraping';
44

@@ -15,6 +15,10 @@ import type {
1515
* A HTTP client implementation based on the `got-scraping` library.
1616
*/
1717
export class GotScrapingHttpClient implements BaseHttpClient {
18+
constructor() {
19+
warnIfBunRuntime();
20+
}
21+
1822
/**
1923
* @inheritDoc
2024
*/

packages/utils/src/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ export * from './internals/iterables';
1313
export * from './internals/robots';
1414
export * from './internals/sitemap';
1515
export * from './internals/url';
16+
export * from './internals/runtime';
1617

1718
export { getCurrentCpuTicksV2 } from './internals/systemInfoV2/cpu-info';
1819
export { getMemoryInfoV2 } from './internals/systemInfoV2/memory-info';
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
let warnedAboutBun = false;
2+
3+
/**
4+
* Detects whether the current process is running under the Bun runtime.
5+
*/
6+
export function isBunRuntime(): boolean {
7+
return typeof globalThis !== 'undefined' && 'Bun' in globalThis;
8+
}
9+
10+
/**
11+
* Logs a one-time warning when `got-scraping` based HTTP client is used under Bun.
12+
* Bun does not fully support `got-scraping`'s tunnel mechanism; users should
13+
* switch to `ImpitHttpClient` from `@crawlee/impit-client` instead.
14+
*/
15+
export function warnIfBunRuntime(logger?: { warning(msg: string, data?: Record<string, unknown>): void }): void {
16+
if (!isBunRuntime() || warnedAboutBun) return;
17+
warnedAboutBun = true;
18+
19+
const message =
20+
'Detected Bun runtime. GotScrapingHttpClient is not fully compatible with Bun — ' +
21+
'proxy tunneling and some Node.js stream APIs may not work. ' +
22+
'Use ImpitHttpClient from @crawlee/impit-client instead: ' +
23+
'new CheerioCrawler({ httpClient: new ImpitHttpClient() })';
24+
25+
if (logger) {
26+
logger.warning(message);
27+
} else {
28+
console.warn(`[crawlee] ${message}`);
29+
}
30+
}
31+
32+
/** @internal — reset for testing */
33+
export function _resetBunWarning(): void {
34+
warnedAboutBun = false;
35+
}

test/core/runtime.test.ts

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import { _resetBunWarning, isBunRuntime, warnIfBunRuntime } from '@crawlee/utils';
2+
3+
afterEach(() => {
4+
_resetBunWarning();
5+
delete (globalThis as any).Bun;
6+
});
7+
8+
describe('isBunRuntime', () => {
9+
test('returns false when Bun global is not present', () => {
10+
delete (globalThis as any).Bun;
11+
expect(isBunRuntime()).toBe(false);
12+
});
13+
14+
test('returns true when Bun global is present', () => {
15+
(globalThis as any).Bun = { version: '1.0.0' };
16+
expect(isBunRuntime()).toBe(true);
17+
});
18+
});
19+
20+
describe('warnIfBunRuntime', () => {
21+
test('does nothing when not running under Bun', () => {
22+
delete (globalThis as any).Bun;
23+
const logger = { warning: vitest.fn() };
24+
warnIfBunRuntime(logger);
25+
expect(logger.warning).not.toHaveBeenCalled();
26+
});
27+
28+
test('calls logger.warning when running under Bun', () => {
29+
(globalThis as any).Bun = { version: '1.0.0' };
30+
const logger = { warning: vitest.fn() };
31+
warnIfBunRuntime(logger);
32+
expect(logger.warning).toHaveBeenCalledTimes(1);
33+
expect(logger.warning).toHaveBeenCalledWith(expect.stringContaining('ImpitHttpClient'));
34+
});
35+
36+
test('only warns once even if called multiple times', () => {
37+
(globalThis as any).Bun = { version: '1.0.0' };
38+
const logger = { warning: vitest.fn() };
39+
warnIfBunRuntime(logger);
40+
warnIfBunRuntime(logger);
41+
warnIfBunRuntime(logger);
42+
expect(logger.warning).toHaveBeenCalledTimes(1);
43+
});
44+
45+
test('falls back to console.warn when no logger is provided', () => {
46+
(globalThis as any).Bun = { version: '1.0.0' };
47+
const spy = vitest.spyOn(console, 'warn').mockImplementation(() => {});
48+
warnIfBunRuntime();
49+
expect(spy).toHaveBeenCalledTimes(1);
50+
expect(spy).toHaveBeenCalledWith(expect.stringContaining('ImpitHttpClient'));
51+
});
52+
});

0 commit comments

Comments
 (0)