fastokens/Cargo.toml at main · crusoecloud/fastokens · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
[workspace]
members = [".", "python"]

[workspace.package]
version = "0.1.1"
edition = "2024"
license = "Apache-2.0"
repository = "https://github.com/Atero-ai/fastokens"
homepage = "https://github.com/Atero-ai/fastokens"
documentation = "https://github.com/Atero-ai/fastokens/blob/main/README.md"

[package]
name = "fastokens"
version.workspace = true
edition.workspace = true
description = "Fast Tokenizer"
license.workspace = true
repository.workspace = true
homepage.workspace = true
documentation.workspace = true

[dependencies]
daachorse = "1.0.0"
fancy-regex = "0.17.0"
memchr = "2"
pcre2 = "0.2"
hf-hub = { version = "0.4.3", default-features = false, features = ["rustls-tls", "ureq"] }
rayon = "1.11.0"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.149"
strum = { version = "0.27.2", features = ["derive"] }
thiserror = "2.0.18"
icu_normalizer = "2"

[dev-dependencies]
anyhow = "1"
clap = { version = "4.5.58", features = ["derive"] }
csv = "1.4.0"
indicatif = "0.18.4"
tokenizers = { version = "0.22.2", features = ["http"] }