Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion examples/SO100/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,14 @@ Visualize it with this [link](https://huggingface.co/spaces/lerobot/visualize_da
uv run --project scripts/lerobot_conversion \
python scripts/lerobot_conversion/convert_v3_to_v2.py \
--repo-id izuluaga/finish_sandwich \
--root examples/SO100/finish_sandwich_lerobot
--root examples/SO100/finish_sandwich_lerobot \
--in-place
```

> `--in-place` replaces the downloaded v3.0 dataset with the converted v2.1 one, so
> the paths below stay the same. The original v3.0 dataset is kept in a sibling
> `finish_sandwich_backup_v3.0` directory.

Then copy the `modality.json` file into the dataset's `meta/` directory:
```bash
cp examples/SO100/modality.json examples/SO100/finish_sandwich_lerobot/izuluaga/finish_sandwich/meta/modality.json
Expand Down
5 changes: 5 additions & 0 deletions scripts/lerobot_conversion/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ Inside the uv environment, run:
python convert_v3_to_v2.py --repo-id BobShan/double_folding_towel_v3.0
```

The converted v2.1 dataset is written next to the source dataset with a `_v2.1`
suffix (or to `--output` if given); the source dataset is left untouched. Pass
`--in-place` to replace the source instead; the original v3.0 dataset is then
kept in a sibling directory with a `_backup_v3.0` suffix.

> **Note:** You may need to install lerobot with `GIT_LFS_SKIP_SMUDGE=1`:
>
> ```bash
Expand Down
97 changes: 86 additions & 11 deletions scripts/lerobot_conversion/convert_v3_to_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@
Usage examples
--------------

- Convert a lerobot v3.0 dataset that already exists locally (The v3.0 path will be overwritten by the v2.1 path and a folder with the suffix _v30 will be created containing the original v3.0 dataset)
- Convert a lerobot v3.0 dataset that already exists locally. The converted v2.1
dataset is written to `--output` (default: a sibling directory with a `_v2.1`
suffix) and the source dataset is left untouched. Pass `--in-place` to replace
the source instead; the original v3.0 dataset is then kept in a sibling
directory with a `_backup_v3.0` suffix.

- This needs lerobot version atleast after commit f55c6e89f.

Expand Down Expand Up @@ -483,13 +487,65 @@ def copy_ancillary_directories(root: Path, new_root: Path) -> None:
shutil.copytree(source, new_root / subdir, dirs_exist_ok=True)


def resolve_output_roots(
root: Path,
output: str | Path | None,
in_place: bool,
) -> tuple[Path, Path | None]:
"""Resolve and validate the destination directories for a conversion.

Returns the directory the converted v2.1 dataset is written to and, when
``in_place`` is requested, the directory the original v3.0 dataset is kept in.
Raises instead of silently overwriting existing data.
"""

if output is not None and in_place:
raise ValueError("--output and --in-place are mutually exclusive.")

new_root = root.parent / f"{root.name}_{V21}" if output is None else Path(output)

root_resolved = root.resolve()
new_root_resolved = new_root.resolve()
if new_root_resolved == root_resolved or root_resolved in new_root_resolved.parents:
raise ValueError(
f"Output directory '{new_root}' would overwrite the source dataset at '{root}'. "
"Pass a different --output, or use --in-place to replace the source explicitly."
)
if new_root.exists():
raise FileExistsError(
f"Output directory already exists: {new_root}. Remove it or pass a different --output."
)

backup_root = None
if in_place:
backup_root = root.parent / f"{root.name}_backup_{V30}"
if backup_root.exists():
raise FileExistsError(
f"Backup directory already exists: {backup_root}. "
"Remove it before converting in place again."
)

return new_root, backup_root


def convert_dataset(
repo_id: str,
root: str | Path | None = None,
output: str | Path | None = None,
in_place: bool = False,
force_conversion: bool = False,
) -> None:
root = HF_LEROBOT_HOME / repo_id if root is None else Path(root) / repo_id

new_root, backup_root = resolve_output_roots(root, output, in_place)
if in_place:
logging.warning(
"--in-place enabled: %s will be replaced by the converted v2.1 dataset; "
"the original v3.0 dataset will be kept at %s",
root,
backup_root,
)

if root.exists() and force_conversion:
logging.info("--force-conversion enabled: removing existing snapshot at %s", root)
shutil.rmtree(root)
Expand All @@ -506,14 +562,6 @@ def convert_dataset(
video_keys = [key for key, ft in info["features"].items() if ft.get("dtype") == "video"]
chunks_size = info.get("chunks_size", DEFAULT_CHUNK_SIZE)

backup_root = root.parent / f"{root.name}_{V30}"
new_root = root.parent / f"{root.name}_{V21}"

if backup_root.is_dir():
shutil.rmtree(backup_root)
if new_root.is_dir():
shutil.rmtree(new_root)

new_root.mkdir(parents=True, exist_ok=True)

convert_info(root, new_root, episode_records, video_keys)
Expand All @@ -524,8 +572,20 @@ def convert_dataset(
convert_episodes_metadata(new_root, episode_records)
copy_ancillary_directories(root, new_root)

shutil.move(str(root), str(backup_root))
shutil.move(str(new_root), str(root))
if in_place:
shutil.move(str(root), str(backup_root))
shutil.move(str(new_root), str(root))
logging.info(
"Replaced %s with the converted v2.1 dataset; the original v3.0 dataset is kept at %s",
root,
backup_root,
)
else:
logging.info(
"Converted v2.1 dataset written to %s; the source dataset at %s was left untouched",
new_root,
root,
)


def parse_args() -> argparse.Namespace:
Expand All @@ -542,6 +602,21 @@ def parse_args() -> argparse.Namespace:
default=None,
help="Local directory under which the dataset should be stored.",
)
parser.add_argument(
"--output",
type=str,
default=None,
help="Directory the converted v2.1 dataset is written to. Defaults to a sibling "
"of the source dataset with a `_v2.1` suffix. The source dataset is never "
"modified unless --in-place is passed.",
)
parser.add_argument(
"--in-place",
action="store_true",
help="Replace the source dataset with the converted v2.1 dataset. The original "
"v3.0 dataset is kept in a sibling directory with a `_backup_v3.0` suffix. "
"Mutually exclusive with --output.",
)
parser.add_argument(
"--force-conversion",
action="store_true",
Expand Down
139 changes: 139 additions & 0 deletions tests/scripts/test_convert_v3_to_v2.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for the non-destructive output handling of ``convert_v3_to_v2.py``."""

from __future__ import annotations

from pathlib import Path
import sys
import types

import pytest


pytest.importorskip("pyarrow")


def _install_lerobot_stubs() -> list[str]:
"""Install just enough fake ``lerobot`` modules to import the conversion script.

The conversion script lives in its own subproject with a real ``lerobot``
dependency; the destination-resolution logic under test does not need it.
"""

stub_attrs: dict[str, dict[str, object]] = {
"lerobot": {},
"lerobot.datasets": {},
"lerobot.datasets.utils": {
"DEFAULT_CHUNK_SIZE": 1000,
"DEFAULT_DATA_PATH": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet",
"DEFAULT_VIDEO_PATH": (
"videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4"
),
"EPISODES_DIR": "meta/episodes",
"LEGACY_EPISODES_PATH": "meta/episodes.jsonl",
"LEGACY_EPISODES_STATS_PATH": "meta/episodes_stats.jsonl",
"LEGACY_TASKS_PATH": "meta/tasks.jsonl",
"load_info": lambda *args, **kwargs: None,
"load_tasks": lambda *args, **kwargs: None,
"serialize_dict": lambda *args, **kwargs: None,
"unflatten_dict": lambda *args, **kwargs: None,
"write_info": lambda *args, **kwargs: None,
},
"lerobot.utils": {},
"lerobot.utils.constants": {"HF_LEROBOT_HOME": Path("unused")},
"lerobot.utils.utils": {"init_logging": lambda *args, **kwargs: None},
}

installed: list[str] = []
for name, attrs in stub_attrs.items():
if name in sys.modules:
continue
module = types.ModuleType(name)
for key, value in attrs.items():
setattr(module, key, value)
sys.modules[name] = module
installed.append(name)
return installed


_stub_names = _install_lerobot_stubs()
try:
from scripts.lerobot_conversion.convert_v3_to_v2 import resolve_output_roots
finally:
for _name in _stub_names:
sys.modules.pop(_name, None)


def test_default_output_is_sibling_with_v21_suffix(tmp_path: Path) -> None:
root = tmp_path / "dataset"
root.mkdir()

new_root, backup_root = resolve_output_roots(root, None, in_place=False)

assert new_root == tmp_path / "dataset_v2.1"
assert backup_root is None


def test_refuses_output_equal_to_source(tmp_path: Path) -> None:
root = tmp_path / "dataset"
root.mkdir()

with pytest.raises(ValueError, match="overwrite the source"):
resolve_output_roots(root, root, in_place=False)


def test_refuses_output_inside_source(tmp_path: Path) -> None:
root = tmp_path / "dataset"
root.mkdir()

with pytest.raises(ValueError, match="overwrite the source"):
resolve_output_roots(root, root / "v2", in_place=False)


def test_refuses_existing_output_directory(tmp_path: Path) -> None:
root = tmp_path / "dataset"
root.mkdir()
(tmp_path / "dataset_v2.1").mkdir()

with pytest.raises(FileExistsError, match="Output directory already exists"):
resolve_output_roots(root, None, in_place=False)


def test_in_place_returns_backup_root(tmp_path: Path) -> None:
root = tmp_path / "dataset"
root.mkdir()

new_root, backup_root = resolve_output_roots(root, None, in_place=True)

assert new_root == tmp_path / "dataset_v2.1"
assert backup_root == tmp_path / "dataset_backup_v3.0"


def test_in_place_refuses_existing_backup(tmp_path: Path) -> None:
root = tmp_path / "dataset"
root.mkdir()
(tmp_path / "dataset_backup_v3.0").mkdir()

with pytest.raises(FileExistsError, match="Backup directory already exists"):
resolve_output_roots(root, None, in_place=True)


def test_output_and_in_place_are_mutually_exclusive(tmp_path: Path) -> None:
root = tmp_path / "dataset"

with pytest.raises(ValueError, match="mutually exclusive"):
resolve_output_roots(root, tmp_path / "out", in_place=True)