crawlberg/Cargo.toml at main · xberg-io/crawlberg · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
[workspace]
members = ["crates/*", "packages/dart/rust", "packages/swift/rust", "tools/benchmark-harness"]
exclude = [
    ".claude",
    "e2e/rust",
    "packages/elixir/native/crawlberg_nif",
    "packages/ruby/ext/crawlberg_rb",
    "packages/ruby/ext/src_rb",
    "packages/ruby/tmp",
    "packages/ruby/vendor",
]
resolver = "2"

[workspace.package]
version = "1.0.2"
edition = "2024"
authors = ["Na'aman Hirschfeld <naaman@xberg.io>"]
license = "MIT"
repository = "https://github.com/xberg-io/crawlberg"
homepage = "https://xberg.io"
documentation = "https://docs.rs/crawlberg"
readme = "README.md"
rust-version = "1.91"
keywords = ["crawl", "scrape", "web", "spider", "rust"]
categories = ["web-programming", "network-programming"]

[workspace.lints.rust]
unsafe_code = "warn"
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(alef)'] }

[workspace.lints.clippy]
all = { level = "warn", priority = -1 }

[workspace.dependencies]
ahash = "0.8"
aho-corasick = "1"
anyhow = "1"
arc-swap = "1"
async-trait = "0.1"
axum = "0.8"
base64 = "0.22"
blake3 = "1"
camino = "1.2"
chromiumoxide = { version = "0.9", default-features = false }
chrono = { version = "0.4", features = ["serde"] }
clap = { version = "4", features = ["derive"] }
crawlberg-browser = { path = "crates/crawlberg-browser", version = "1.0.2" }
crawlberg-bypass = { path = "crates/crawlberg-bypass", version = "1.0.2" }
criterion = { version = "0.8", features = ["html_reports"] }
cssparser = "0.37"
dashmap = "6"
deno_core = "0.406"
deno_error = "0.7"
dirs = "6"
encoding_rs = "0.8"

# Binding framework dependencies
ext-php-rs = "0.15.15"
flate2 = "1"
futures = "0.3"
html-to-markdown-rs = { version = "3.8", features = ["full"] }
html5ever = "0.39"
ipnet = "2"
itertools = "0.15"
libc = "0.2"
liter-llm = { version = "1.9", features = ["native-http"] }
markup5ever = "0.39"
memchr = "2"
minijinja = "2"
napi = { version = "3", default-features = false, features = ["napi8", "tokio_rt"] }
napi-derive = "3"
notify = "8"
# OpenTelemetry stack. Pinned to match xberg-enterprise's workspace pins so cloud
# can consume crawlberg spans/metrics without a duplicate global provider.
# Held at 0.32: `tracing-opentelemetry` 0.32.1 still depends on `opentelemetry`
# 0.31 internally, so bumping us further splits the dep graph and breaks
# cross-service propagator injection (see xberg-enterprise Cargo.toml:89-91).
opentelemetry = { version = "0.32", features = ["trace", "metrics"] }
opentelemetry-otlp = { version = "0.32", features = ["grpc-tonic", "trace", "metrics"] }
opentelemetry-semantic-conventions = "0.32"
opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
png = "0.18"
precomputed-hash = "0.1"
proptest = "1"
pyo3 = { version = "0.29", features = ["abi3-py310"] }
pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] }
quick-xml = "0.41"
rayon = "1"
regex = "1"
reqwest = { version = "0.13", features = ["cookies", "gzip", "brotli"] }
rmcp = { version = "2", features = ["server", "macros", "transport-io", "transport-streamable-http-server"] }
schemars = "1.2"
selectors = "0.38"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
servo_arc = "0.4"
sha2 = "0.11"
tempfile = "3"
thiserror = "2"
tl = { package = "astral-tl", version = "0.7" }
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
toml = "1.1"
tower = { version = "0.5", features = ["timeout", "retry", "limit", "util", "make"] }
tower-http = { version = "0.7", features = [
    "cors",
    "compression-full",
    "trace",
    "limit",
    "catch-panic",
    "request-id",
    "sensitive-headers",
] }
tracing = "0.1"
tracing-opentelemetry = "0.33"
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }
url = "2"
utoipa = { version = "5", features = ["axum_extras"] }
uuid = { version = "1.23", features = ["v4"] }
walkdir = "2.5"
wasm-bindgen = "0.2"
wasm-bindgen-futures = "0.4"