diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 13ff02469..ad9b9c123 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -305,6 +305,25 @@ We don't use the package versions (in pyproject.toml files), no need to update t All the contributions should go through a pull request. The pull requests must be "squashed" (ie: one commit per pull request). +## Update common dependencies + +1. Go to `libs/libcommon` + +2. Update dependencies in `pyproject.toml`. + +3. Run a CLI on all projects that depend on libcommon: + +```bash +>>> poetry run cli_all_projects +Multi CLI for projects: ... +``` + +4. Then you can update all dependencies using + +```bash +>>> poetry lock +``` + ## GitHub Actions You can use [act](https://github.com/nektos/act) to test the GitHub Actions (see [.github/workflows/](.github/workflows/)) locally. It reduces the retroaction loop when working on the GitHub Actions, avoid polluting the branches with empty pushes only meant to trigger the CI, and allows to only run specific actions. diff --git a/front/admin_ui/poetry.lock b/front/admin_ui/poetry.lock index b41374828..8f8dad50a 100644 --- a/front/admin_ui/poetry.lock +++ b/front/admin_ui/poetry.lock @@ -716,8 +716,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "dill" @@ -1524,7 +1524,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/jobs/cache_maintenance/poetry.lock b/jobs/cache_maintenance/poetry.lock index 7eddfa308..2e125957a 100644 --- a/jobs/cache_maintenance/poetry.lock +++ b/jobs/cache_maintenance/poetry.lock @@ -684,8 +684,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -1204,7 +1204,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/jobs/mongodb_migration/poetry.lock b/jobs/mongodb_migration/poetry.lock index 1dd3064bf..3c9b51b0a 100644 --- a/jobs/mongodb_migration/poetry.lock +++ b/jobs/mongodb_migration/poetry.lock @@ -684,8 +684,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -1204,7 +1204,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/libs/libapi/poetry.lock b/libs/libapi/poetry.lock index 58930956a..5f6cb69c9 100644 --- a/libs/libapi/poetry.lock +++ b/libs/libapi/poetry.lock @@ -684,8 +684,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -1223,7 +1223,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/libs/libcommon/poetry.lock b/libs/libcommon/poetry.lock index d5d9aadc3..304fd4642 100644 --- a/libs/libcommon/poetry.lock +++ b/libs/libcommon/poetry.lock @@ -678,7 +678,7 @@ name = "datasets" version = "4.5.1.dev0" description = "HuggingFace community-driven open-source library of datasets" optional = false -python-versions = ">=3.9.0" +python-versions = ">=3.10.0" groups = ["main"] files = [] develop = false @@ -719,8 +719,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -4685,4 +4685,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "3.12.11" -content-hash = "e727583779e5192047636267fd507cadc0180248bfa7687c7cf5ba3504df9c1c" +content-hash = "4e8234aa7f49205e592713c9b2d4d47029d25da3dfb6bb1597d8c0e68d3e200d" diff --git a/libs/libcommon/pyproject.toml b/libs/libcommon/pyproject.toml index 28de22ccd..cb47c8181 100644 --- a/libs/libcommon/pyproject.toml +++ b/libs/libcommon/pyproject.toml @@ -10,7 +10,7 @@ python = "3.12.11" anyio = ">=3.4.0,<5" appdirs = "^1.4.4" cryptography = "^43.0.1" -datasets = {git="https://github.com/huggingface/datasets.git", rev="f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git="https://github.com/huggingface/datasets.git", rev="6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" @@ -73,6 +73,9 @@ validators = "^0.22.0" build-backend = "poetry.core.masonry.api" requires = ["poetry-core>=1.0.0"] +[tool.poetry.scripts] +cli_all_projects = 'libcommon.scripts:cli_all_projects' + [tool.pytest.ini_options] filterwarnings = ["ignore::DeprecationWarning"] markers = [ diff --git a/libs/libcommon/src/libcommon/scripts.py b/libs/libcommon/src/libcommon/scripts.py new file mode 100644 index 000000000..f5e418691 --- /dev/null +++ b/libs/libcommon/src/libcommon/scripts.py @@ -0,0 +1,20 @@ +import subprocess # nosec +from pathlib import Path + + +def cli_all_projects() -> None: + all_projects_dir = Path(__file__).resolve().parents[-5] + projects = [ + p + for d in all_projects_dir.glob("*") + if not d.name.startswith(".") and d.is_dir() + for p in list(d.glob("*")) + [d] + if (p / "pyproject.toml").exists() + ] + projects.sort(key=lambda p: (p.name != "libcommon", "libs" not in p.parts, p)) + + print(f"Multi CLI for projects: {', '.join(str(project) for project in projects)}") + while (command := input(">>> ")) not in ["exit", "exit()"]: + for project in projects: + print(f"cd {project}") + subprocess.run(command.split(" "), cwd=project) # nosec diff --git a/services/admin/poetry.lock b/services/admin/poetry.lock index 03881861f..282cd217a 100644 --- a/services/admin/poetry.lock +++ b/services/admin/poetry.lock @@ -684,8 +684,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -1227,7 +1227,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/api/poetry.lock b/services/api/poetry.lock index 8854d40f3..c6d5a5884 100644 --- a/services/api/poetry.lock +++ b/services/api/poetry.lock @@ -684,8 +684,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -1264,7 +1264,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/rows/poetry.lock b/services/rows/poetry.lock index 3c3dcaa6f..42ba5e04f 100644 --- a/services/rows/poetry.lock +++ b/services/rows/poetry.lock @@ -705,8 +705,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -1285,7 +1285,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/search/poetry.lock b/services/search/poetry.lock index bd6ad63f2..dff9cef89 100644 --- a/services/search/poetry.lock +++ b/services/search/poetry.lock @@ -684,8 +684,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -1264,7 +1264,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/sse-api/poetry.lock b/services/sse-api/poetry.lock index a2da4a0d4..9ca3917df 100644 --- a/services/sse-api/poetry.lock +++ b/services/sse-api/poetry.lock @@ -684,8 +684,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -1295,7 +1295,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/webhook/poetry.lock b/services/webhook/poetry.lock index a1379639a..5723bd93e 100644 --- a/services/webhook/poetry.lock +++ b/services/webhook/poetry.lock @@ -684,8 +684,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -1264,7 +1264,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/worker/poetry.lock b/services/worker/poetry.lock index b1254f1c3..3a884d29e 100644 --- a/services/worker/poetry.lock +++ b/services/worker/poetry.lock @@ -986,7 +986,7 @@ name = "datasets" version = "4.5.1.dev0" description = "HuggingFace community-driven open-source library of datasets" optional = false -python-versions = ">=3.9.0" +python-versions = ">=3.10.0" groups = ["main"] files = [] develop = false @@ -1027,8 +1027,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets.git" -reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" -resolved_reference = "f15c59970fef04e277a93b9c262ccdc7bd8038a4" +reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" +resolved_reference = "6db5f805a444a767726d58d5dd1cfaa25fc852c0" [[package]] name = "defusedxml" @@ -1701,7 +1701,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f15c59970fef04e277a93b9c262ccdc7bd8038a4", extras = ["vision"]} +datasets = {git = "https://github.com/huggingface/datasets.git", rev = "6db5f805a444a767726d58d5dd1cfaa25fc852c0", extras = ["vision"]} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/worker/tests/fixtures/hub.py b/services/worker/tests/fixtures/hub.py index 1b53559e6..681dd700b 100644 --- a/services/worker/tests/fixtures/hub.py +++ b/services/worker/tests/fixtures/hub.py @@ -566,9 +566,9 @@ def create_dataset_info_response_for_audio(dataset: str, config: str) -> Any: "config_name": config, "dataset_name": dataset_name, "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, - "splits": {"train": {"name": "train", "num_bytes": 15448, "num_examples": 1, "dataset_name": None}}, + "splits": {"train": {"name": "train", "num_bytes": 15414, "num_examples": 1, "dataset_name": None}}, "download_size": AUDIO_PARQUET_SIZE, - "dataset_size": 15448, + "dataset_size": 15414, } @@ -648,7 +648,7 @@ def create_parquet_and_info_response( CSV_PARQUET_SIZE = 1730 PARTIAL_CSV_PARQUET_SIZE = 5806 -AUDIO_PARQUET_SIZE = 16546 +AUDIO_PARQUET_SIZE = 16477 BIG_PARQUET_FILE = 18418 DATA_cols = {