diff --git a/examples/SO100/README.md b/examples/SO100/README.md index afd5cc1ef..348e1d111 100644 --- a/examples/SO100/README.md +++ b/examples/SO100/README.md @@ -17,9 +17,14 @@ Visualize it with this [link](https://huggingface.co/spaces/lerobot/visualize_da uv run --project scripts/lerobot_conversion \ python scripts/lerobot_conversion/convert_v3_to_v2.py \ --repo-id izuluaga/finish_sandwich \ - --root examples/SO100/finish_sandwich_lerobot + --root examples/SO100/finish_sandwich_lerobot \ + --in-place ``` +> `--in-place` replaces the downloaded v3.0 dataset with the converted v2.1 one, so +> the paths below stay the same. The original v3.0 dataset is kept in a sibling +> `finish_sandwich_backup_v3.0` directory. + Then copy the `modality.json` file into the dataset's `meta/` directory: ```bash cp examples/SO100/modality.json examples/SO100/finish_sandwich_lerobot/izuluaga/finish_sandwich/meta/modality.json diff --git a/scripts/lerobot_conversion/README.md b/scripts/lerobot_conversion/README.md index 6f24e81af..d77b93de2 100644 --- a/scripts/lerobot_conversion/README.md +++ b/scripts/lerobot_conversion/README.md @@ -21,6 +21,11 @@ Inside the uv environment, run: python convert_v3_to_v2.py --repo-id BobShan/double_folding_towel_v3.0 ``` +The converted v2.1 dataset is written next to the source dataset with a `_v2.1` +suffix (or to `--output` if given); the source dataset is left untouched. Pass +`--in-place` to replace the source instead; the original v3.0 dataset is then +kept in a sibling directory with a `_backup_v3.0` suffix. + > **Note:** You may need to install lerobot with `GIT_LFS_SKIP_SMUDGE=1`: > > ```bash diff --git a/scripts/lerobot_conversion/convert_v3_to_v2.py b/scripts/lerobot_conversion/convert_v3_to_v2.py index 0d6dc84ef..20d207e13 100644 --- a/scripts/lerobot_conversion/convert_v3_to_v2.py +++ b/scripts/lerobot_conversion/convert_v3_to_v2.py @@ -18,7 +18,11 @@ Usage examples -------------- -- Convert a lerobot v3.0 dataset that already exists locally (The v3.0 path will be overwritten by the v2.1 path and a folder with the suffix _v30 will be created containing the original v3.0 dataset) +- Convert a lerobot v3.0 dataset that already exists locally. The converted v2.1 + dataset is written to `--output` (default: a sibling directory with a `_v2.1` + suffix) and the source dataset is left untouched. Pass `--in-place` to replace + the source instead; the original v3.0 dataset is then kept in a sibling + directory with a `_backup_v3.0` suffix. - This needs lerobot version atleast after commit f55c6e89f. @@ -483,13 +487,65 @@ def copy_ancillary_directories(root: Path, new_root: Path) -> None: shutil.copytree(source, new_root / subdir, dirs_exist_ok=True) +def resolve_output_roots( + root: Path, + output: str | Path | None, + in_place: bool, +) -> tuple[Path, Path | None]: + """Resolve and validate the destination directories for a conversion. + + Returns the directory the converted v2.1 dataset is written to and, when + ``in_place`` is requested, the directory the original v3.0 dataset is kept in. + Raises instead of silently overwriting existing data. + """ + + if output is not None and in_place: + raise ValueError("--output and --in-place are mutually exclusive.") + + new_root = root.parent / f"{root.name}_{V21}" if output is None else Path(output) + + root_resolved = root.resolve() + new_root_resolved = new_root.resolve() + if new_root_resolved == root_resolved or root_resolved in new_root_resolved.parents: + raise ValueError( + f"Output directory '{new_root}' would overwrite the source dataset at '{root}'. " + "Pass a different --output, or use --in-place to replace the source explicitly." + ) + if new_root.exists(): + raise FileExistsError( + f"Output directory already exists: {new_root}. Remove it or pass a different --output." + ) + + backup_root = None + if in_place: + backup_root = root.parent / f"{root.name}_backup_{V30}" + if backup_root.exists(): + raise FileExistsError( + f"Backup directory already exists: {backup_root}. " + "Remove it before converting in place again." + ) + + return new_root, backup_root + + def convert_dataset( repo_id: str, root: str | Path | None = None, + output: str | Path | None = None, + in_place: bool = False, force_conversion: bool = False, ) -> None: root = HF_LEROBOT_HOME / repo_id if root is None else Path(root) / repo_id + new_root, backup_root = resolve_output_roots(root, output, in_place) + if in_place: + logging.warning( + "--in-place enabled: %s will be replaced by the converted v2.1 dataset; " + "the original v3.0 dataset will be kept at %s", + root, + backup_root, + ) + if root.exists() and force_conversion: logging.info("--force-conversion enabled: removing existing snapshot at %s", root) shutil.rmtree(root) @@ -506,14 +562,6 @@ def convert_dataset( video_keys = [key for key, ft in info["features"].items() if ft.get("dtype") == "video"] chunks_size = info.get("chunks_size", DEFAULT_CHUNK_SIZE) - backup_root = root.parent / f"{root.name}_{V30}" - new_root = root.parent / f"{root.name}_{V21}" - - if backup_root.is_dir(): - shutil.rmtree(backup_root) - if new_root.is_dir(): - shutil.rmtree(new_root) - new_root.mkdir(parents=True, exist_ok=True) convert_info(root, new_root, episode_records, video_keys) @@ -524,8 +572,20 @@ def convert_dataset( convert_episodes_metadata(new_root, episode_records) copy_ancillary_directories(root, new_root) - shutil.move(str(root), str(backup_root)) - shutil.move(str(new_root), str(root)) + if in_place: + shutil.move(str(root), str(backup_root)) + shutil.move(str(new_root), str(root)) + logging.info( + "Replaced %s with the converted v2.1 dataset; the original v3.0 dataset is kept at %s", + root, + backup_root, + ) + else: + logging.info( + "Converted v2.1 dataset written to %s; the source dataset at %s was left untouched", + new_root, + root, + ) def parse_args() -> argparse.Namespace: @@ -542,6 +602,21 @@ def parse_args() -> argparse.Namespace: default=None, help="Local directory under which the dataset should be stored.", ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Directory the converted v2.1 dataset is written to. Defaults to a sibling " + "of the source dataset with a `_v2.1` suffix. The source dataset is never " + "modified unless --in-place is passed.", + ) + parser.add_argument( + "--in-place", + action="store_true", + help="Replace the source dataset with the converted v2.1 dataset. The original " + "v3.0 dataset is kept in a sibling directory with a `_backup_v3.0` suffix. " + "Mutually exclusive with --output.", + ) parser.add_argument( "--force-conversion", action="store_true", diff --git a/tests/scripts/test_convert_v3_to_v2.py b/tests/scripts/test_convert_v3_to_v2.py new file mode 100644 index 000000000..357005fcf --- /dev/null +++ b/tests/scripts/test_convert_v3_to_v2.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the non-destructive output handling of ``convert_v3_to_v2.py``.""" + +from __future__ import annotations + +from pathlib import Path +import sys +import types + +import pytest + + +pytest.importorskip("pyarrow") + + +def _install_lerobot_stubs() -> list[str]: + """Install just enough fake ``lerobot`` modules to import the conversion script. + + The conversion script lives in its own subproject with a real ``lerobot`` + dependency; the destination-resolution logic under test does not need it. + """ + + stub_attrs: dict[str, dict[str, object]] = { + "lerobot": {}, + "lerobot.datasets": {}, + "lerobot.datasets.utils": { + "DEFAULT_CHUNK_SIZE": 1000, + "DEFAULT_DATA_PATH": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet", + "DEFAULT_VIDEO_PATH": ( + "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4" + ), + "EPISODES_DIR": "meta/episodes", + "LEGACY_EPISODES_PATH": "meta/episodes.jsonl", + "LEGACY_EPISODES_STATS_PATH": "meta/episodes_stats.jsonl", + "LEGACY_TASKS_PATH": "meta/tasks.jsonl", + "load_info": lambda *args, **kwargs: None, + "load_tasks": lambda *args, **kwargs: None, + "serialize_dict": lambda *args, **kwargs: None, + "unflatten_dict": lambda *args, **kwargs: None, + "write_info": lambda *args, **kwargs: None, + }, + "lerobot.utils": {}, + "lerobot.utils.constants": {"HF_LEROBOT_HOME": Path("unused")}, + "lerobot.utils.utils": {"init_logging": lambda *args, **kwargs: None}, + } + + installed: list[str] = [] + for name, attrs in stub_attrs.items(): + if name in sys.modules: + continue + module = types.ModuleType(name) + for key, value in attrs.items(): + setattr(module, key, value) + sys.modules[name] = module + installed.append(name) + return installed + + +_stub_names = _install_lerobot_stubs() +try: + from scripts.lerobot_conversion.convert_v3_to_v2 import resolve_output_roots +finally: + for _name in _stub_names: + sys.modules.pop(_name, None) + + +def test_default_output_is_sibling_with_v21_suffix(tmp_path: Path) -> None: + root = tmp_path / "dataset" + root.mkdir() + + new_root, backup_root = resolve_output_roots(root, None, in_place=False) + + assert new_root == tmp_path / "dataset_v2.1" + assert backup_root is None + + +def test_refuses_output_equal_to_source(tmp_path: Path) -> None: + root = tmp_path / "dataset" + root.mkdir() + + with pytest.raises(ValueError, match="overwrite the source"): + resolve_output_roots(root, root, in_place=False) + + +def test_refuses_output_inside_source(tmp_path: Path) -> None: + root = tmp_path / "dataset" + root.mkdir() + + with pytest.raises(ValueError, match="overwrite the source"): + resolve_output_roots(root, root / "v2", in_place=False) + + +def test_refuses_existing_output_directory(tmp_path: Path) -> None: + root = tmp_path / "dataset" + root.mkdir() + (tmp_path / "dataset_v2.1").mkdir() + + with pytest.raises(FileExistsError, match="Output directory already exists"): + resolve_output_roots(root, None, in_place=False) + + +def test_in_place_returns_backup_root(tmp_path: Path) -> None: + root = tmp_path / "dataset" + root.mkdir() + + new_root, backup_root = resolve_output_roots(root, None, in_place=True) + + assert new_root == tmp_path / "dataset_v2.1" + assert backup_root == tmp_path / "dataset_backup_v3.0" + + +def test_in_place_refuses_existing_backup(tmp_path: Path) -> None: + root = tmp_path / "dataset" + root.mkdir() + (tmp_path / "dataset_backup_v3.0").mkdir() + + with pytest.raises(FileExistsError, match="Backup directory already exists"): + resolve_output_roots(root, None, in_place=True) + + +def test_output_and_in_place_are_mutually_exclusive(tmp_path: Path) -> None: + root = tmp_path / "dataset" + + with pytest.raises(ValueError, match="mutually exclusive"): + resolve_output_roots(root, tmp_path / "out", in_place=True)