pyproject.toml

[project]
name = "datatrove"
version = "0.9.0"  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
description = "HuggingFace library to process and filter large amounts of webdata"
readme = "README.md"
authors = [
  {name = "MacroData Inc.", email = "guilherme@macrodata.co"},
  {name = "HuggingFace Inc.", email = "joel@hf.co"}
]
license = {text = "Apache-2.0"}
classifiers = [
  "Intended Audience :: Developers",
  "Intended Audience :: Education",
  "Intended Audience :: Science/Research",
  "License :: OSI Approved :: Apache Software License",
  "Operating System :: OS Independent",
  "Programming Language :: Python :: 3",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: 3.12",
  "Topic :: Scientific/Engineering :: Artificial Intelligence",
]
keywords = ["data", "machine", "learning", "processing"]
requires-python = ">=3.10.0"
dependencies = [
  "dill>=0.3.0",
  "fsspec>=2023.12.2",
  "huggingface-hub>=0.34", # this version enables hf-xet
  "humanize",
  "loguru>=0.7.0",
  "multiprocess",
  "numpy>=2.0.0",
  "tqdm",
]

[project.optional-dependencies]
cli = [
  "rich",
]
io = [
  "faust-cchardet",
  "pyarrow",
  "python-magic",
  "warcio",
  "datasets>=3.1.0",
  "orjson",
  "zstandard",
]
s3 = [
  "s3fs>=2023.12.2",
]
processing = [
  "fasttext-numpy2-wheel",
  "nltk",
  "inscriptis",
  "tldextract",
  "trafilatura>=1.8.0,<1.12.0",
  "tokenizers",
  "ftfy",
  "fasteners",
  "regex",
  "xxhash",
  "pyahocorasick",
]
decont = [
  "lighteval>=0.3.0",
]
multilingual = [
  "spacy[ja]>=3.8",
  "stanza",
  "pyvi", # vietnamese
  "pythainlp", # thai
  "jieba", # chinese
  "indic-nlp-library", # indic languages
  "kiwipiepy<0.22.0", # korean - v0.22.0 introduced issues, pin to previous version
  # urduhack has keras and tensorflow as dependencies and requires a specific version to work...
  "urduhack",
  "tensorflow>=2.16",
  "khmer-nltk", # khmer
  "laonlp", # lao
  "botok", # tibetan languages,
  "pyidaungsu-numpy2", # burmese
]
inference = [
  "datatrove[io]",
  "aiofiles",
  "httpx",
  "aiosqlite",
  "vllm>=0.10.0",
  "sglang",
  "bitsandbytes",
  "numpy>=2.0.0,<2.3", # numba requires numpy<=2.2
  "typer",
  "pyyaml",
  "pandas",
  "transformers",
]
ray = [
  "ray[default]",
]
quality = [
  "ruff>=0.1.5",
]
testing = [
  "datatrove[cli]",
  "datatrove[io]",
  "datatrove[processing]",
  "datatrove[multilingual]",
  "datatrove[s3]",
  "datatrove[ray]",
  "datatrove[inference]",
  "datatrove[decont]",
  # Flask doesn't have correct dependencies on werkzeug, causing issues, thus we pin flask 3.1 (which currently works) to avoid it
  "flask>=3.1.0",
  "pytest",
  "pytest-rerunfailures",
  "pytest-timeout",
  "pytest-xdist",
  "moto[s3,server]",
]
all = [
  "datatrove[quality]",
  "datatrove[testing]",
]
dev = [
  "datatrove[all]",
]

[project.urls]
Repository = "https://github.com/huggingface/datatrove"

[project.scripts]
check_dataset = "datatrove.tools.check_dataset:main"
merge_stats = "datatrove.tools.merge_stats:main"
launch_pickled_pipeline = "datatrove.tools.launch_pickled_pipeline:main"
failed_logs = "datatrove.tools.failed_logs:main"
inspect_data = "datatrove.tools.inspect_data:main"
jobs_status = "datatrove.tools.jobs_status:main"
track_jobs = "datatrove.tools.track_jobs:main"

[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"

[tool.setuptools.packages.find]
where = ["src"]

[tool.setuptools.package-data]
datatrove = [
  "assets/*",
  "pipeline/inference/*.md",
]

[tool.ruff]
lint.ignore = [
  "C901", # `function_name` is too complex
  "E501", # line length violation
]
lint.select = [
  "C",
  "E",
  "F",
  "I",
  "W",
  "NPY201", # numpy 2.0.0
]
line-length = 119

[tool.ruff.lint.per-file-ignores]
"__init__.py" = [
  "F401" # module imported but unused
]

[tool.ruff.lint.isort]
lines-after-imports = 2
known-first-party = [
  "datatrove"
]

[tool.pytest.ini_options]
timeout = 120  # 2 minute timeout for all tests to prevent CI hangs