dataset-viewer/libs/libcommon/pyproject.toml at 6a5295dd50408dca0a72bf0acd48229c49b0e0a4 · huggingface/dataset-viewer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
[tool.poetry]
authors = ["Sylvain Lesage <sylvain.lesage@huggingface.co>"]
description = "Library for utils common to all the services"
name = "libcommon"
version = "0.6.8"
license = "Apache-2.0"

[tool.poetry.dependencies]
python = "3.12.11"
anyio = ">=3.4.0,<5"
appdirs = "^1.4.4"
cryptography = "^43.0.1"
datasets = "^4.8.2"
duckdb = "^1.2.2"
environs = "^14.3.0"
filelock = "^3.18.0"
fsspec = {version = "2024.3.1", extras = ["s3"]}
huggingface-hub = {version = "^0.31.0", extras = ["hf-transfer"]}
mongo-types = "0.15.1"
mongoengine = "^0.29.1"
networkx = "^3.0"
numpy = "^2.3.3"
orjson = "^3.9.15"
pandas = "^2.2.0"
pdfplumber = ">=0.11.4"
pillow = "^10.3.0"
polars = "1.31.0"  # 1.33.1 has issues with datetimes
psutil = "^7.1.0"
pyarrow = "^21.0.0"
pylance = "^1.0.4"
pymongoarrow = "^1.10.0"
pymupdf = "^1.26.1"
pydub = "^0.25.1"
pymongo = { extras = ["srv"], version = "^4.15.1" }
pytz = "^2020.1"
s3fs = "2024.3.1"  # Aligned with fsspec[s3] version
soundfile = ">=0.12.1"
starlette-prometheus = "^0.9.0"
torch = [
    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", markers = "sys_platform == 'linux' and platform_machine != 'aarch64'"},
    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", markers = "sys_platform == 'darwin' and platform_machine != 'arm64'"},
    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", markers = "sys_platform == 'darwin' and platform_machine == 'arm64'"},
    { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", markers = "sys_platform == 'linux' and platform_machine == 'aarch64'"},
]
# torchcodec is not available for aarch64 linux so skip its installation on that platform
# it can prevent some build headaches (especially building linux/aarch64 docker images on
# arm macs) while the test suite should raise an error if any code path requires it on
# that platform
torchcodec = { version = "0.7.0", markers = "sys_platform != 'linux' or platform_machine != 'aarch64'" }
tqdm = "^4.66.3"
async-lru = "^2.0.5"
libviewer = { path = "../libviewer", develop = true }

[tool.poetry.group.dev.dependencies]
aiobotocore = "2.13.0"
bandit = "^1.7.4"
boto3 = "1.34.106"
httpx = "^0.26.0"
moto = "^4.2.8"
mypy = "^1.10.0"
pandas-stubs = "^2.2.0"
pip-audit = "^2.7.3"
pytest = "^8.2.2"
pytest-datadir = "^1.5.0"
pytest-memray = "^1.6.0"
ruff = "^0"
types-aiobotocore = {extras = ["signer"], version = "2.13.0"}
types-psutil = "^7.0.0"
types-pytz = "^2022.1.1"
validators = "^0.22.0"

[build-system]
build-backend = "poetry.core.masonry.api"
requires = ["poetry-core>=1.0.0"]

[tool.poetry.scripts]
cli_all_projects = 'libcommon.scripts:cli_all_projects'

[tool.pytest.ini_options]
filterwarnings = ["ignore::DeprecationWarning"]
markers = [
    "real_dataset: tests on the Hub"
]

[tool.mypy]
strict = true
# allow calling untyped methods in huggingface_hub (eg: DatasetInfo(...))
untyped_calls_exclude = "huggingface_hub"

[[tool.mypy.overrides]]
module = [
    "datasets.*",
    "fitz.*",
    "networkx.*",
    "prometheus_client.*",
    "pyarrow.*",
    "pymongoarrow.*",
    "tqdm.*",
    "fsspec.*",
    "boto3.*",
    "moto.*",
    "aiobotocore.*",
    "requests.*",
    "dateutil.*",
    "torchcodec.*",
    "duckdb.*",
    "libviewer.*"
]
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = [
    "huggingface_hub.*",
]
# allow
#   from huggingface_hub.utils import build_hf_headers
# even if the module does not explicitly exports the method
# https://github.com/huggingface/huggingface_hub/blob/07896ee75b37da0d1744c9d03472485b985b3213/src/huggingface_hub/utils/__init__.py
no_implicit_reexport = false

[tool.ruff]
line-length = 119
src = ["src"]
target-version = "py39"

[tool.ruff.lint]
extend-select = [
    "ARG",  # flake8-unused-arguments
    "I",  # isort
    # flake8-pep585:
    "UP006",  # non-pep585-annotation
    "UP035",  # deprecated-import
]