Merge pull request #50 from AGH-CEAI/feature/cut_xacro_dependency

Cut xacro dependency & download URDF assets from ClearML

Author: macmacal

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- [PR-49](https://github.com/AGH-CEAI/aegis_gym/pull/49) - Utility script for uploading the URDF assets to the ClearML server as a dataset.
 - [PR-48](https://github.com/AGH-CEAI/aegis_gym/pull/48) - Added real robot control via gRPC in RSL-RL Grasp env.
 - [PR-46](https://github.com/AGH-CEAI/aegis_gym/pull/46) - Added TCP-to-object Grasp environment.
 - [PR-40](https://github.com/AGH-CEAI/aegis_gym/pull/40) - Added scene and tool cameras setup for Grasp environment.
@@ -27,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- [PR-49](https://github.com/AGH-CEAI/aegis_gym/pull/49) - The `AegisGrasp`'s rsl_rl robot config accepts ID to download URDF dataset from ClearML (see [aegis_ros PR-95](https://github.com/AGH-CEAI/aegis_ros/pull/95)).
 - [PR-42](https://github.com/AGH-CEAI/aegis_gym/pull/42) - Extracted Grasp environment configs to a new file.
 - [PR-42](https://github.com/AGH-CEAI/aegis_gym/pull/42) - Ported Grasp environment to use `rsl-rl-lib==3.3.0`.
 - [PR-38](https://github.com/AGH-CEAI/aegis_gym/pull/38) - Changed `ur_base` frame to the `world` frame.
@@ -44,6 +46,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+- [PR-49](https://github.com/AGH-CEAI/aegis_gym/pull/49) - Removed automatic URDF generation with `xacro`.
+
 ### Fixed
 
 - [PR-46](https://github.com/AGH-CEAI/aegis_gym/pull/46) - Fixed model sorting and typos.

README.md

@@ -37,6 +37,35 @@ poetry run pytest -v -s
 ```bash
 python3 ./test/sb3_run_train.py
 ```
+
+---
+## URDF model for simulator
+
+### Uploading
+1. Generate standalone URDF model with [aegis_ros/aegis_descrption]() launch command:
+```bash
+ros2 launch aegis_description generate_standalone_urdf.launch.py disable_cell:=true
+```
+Which will generate the whole URDF file with 3D models in a default `~/ceai_ws/aegis_urdf` directory.
+
+2. Run the [`utils/upload_urdf_to_clearml.py`](./utils/upload_urdf_to_clearml.py) script with the following options:
+```bash
+python3 utils/upload_urdf_to_clearml.py ~/ceai_ws/aegis_urdf --name AegisURDFModel --project AEGIS_GRASP --desc "Aegis simulator assets"
+```
+> [!WARNING]
+> **To update the dataset** make sure to add an additional option: `--parent "PREVIOUS_DATASET_ID"`
+
+3. Check the ClearML server's datasets.
+
+### Usage
+
+In the robot's config set the `urdf_model_id` param to the ClearML's dataset ID.
+
+> [!IMPORTANT]
+> In case of failure to obtain the model, the code will try to load URDF model from `~/ceai_ws/aegid_urdf` directory.
+
+
+
 ---
 ## Development notes
 

aegis_gym/rsl/envs/manipulator.py

@@ -1,14 +1,16 @@
+import time
+import warnings
+from pathlib import Path
 from typing import Literal
 
 import torch as th
 import genesis as gs
+from clearml import Dataset
 from genesis.utils.geom import (
     transform_quat_by_quat,
     xyz_to_quat,
 )
 
-from .utils import generate_aegis_urdf
-
 
 class Manipulator:
     def __init__(
@@ -25,10 +27,22 @@ def __init__(
         self._num_envs = num_envs
         self._args = args
 
+        if show_cell:
+            self._urdf_model_id = args["urdf_model_id"]["cell"]
+        else:
+            self._urdf_model_id = args["urdf_model_id"]["no_cell"]
+
+        if self._urdf_model_id:
+            print(
+                f"[GraspEnv::Manipulator] URDF ClearML dataset ID: {self._urdf_model_id}"
+            )
+        self._urdf_path = self._resolve_aegis_urdf()
+        print(f"[GraspEnv::Manipulator] URDF path: {self._urdf_path}")
+
         # == Genesis configurations ==
         material: gs.materials.Rigid = gs.materials.Rigid()
         morph: gs.morphs.URDF = gs.morphs.URDF(
-            file=generate_aegis_urdf(show_cell),
+            file=self._urdf_path,
             fixed=True,
             pos=(0.0, 0.0, 0.0),
             quat=(1.0, 0.0, 0.0, 0.0),
@@ -49,6 +63,41 @@ def __init__(
 
         self._init()
 
+    def _resolve_aegis_urdf(self) -> Path:
+        default_path = Path("~/ceai_ws/aegis_urdf/aegis.urdf").expanduser().resolve()
+
+        if self._urdf_model_id is not None:
+            try:
+                dataset = Dataset.get(dataset_id=self._urdf_model_id)
+                local_path = Path(dataset.get_local_copy())
+            except ValueError:
+                warnings.warn(
+                    "Failed to obtain the dataset: `{e}`. Fallbacking to the default path..."
+                )
+                return default_path
+
+            urdf_files = list(local_path.rglob("*.urdf"))
+            if not urdf_files:
+                raise FileNotFoundError(
+                    f"No URDF file in dataset {self._urdf_model_id}"
+                )
+            if len(urdf_files) > 1:
+                raise RuntimeError(
+                    f"Found {len(urdf_files)} URDF files in dataset {self._urdf_model_id}, expected just one"
+                )
+            return str(urdf_files[0])
+
+        warnings.warn(
+            "There is no given ClearML dataset ID for the URDF assets! Trying to read the default directory in 5s.."
+        )
+        time.sleep(5.0)
+
+        if not default_path.exists():
+            raise FileNotFoundError(
+                f"Couldn't resolve the path to the URDF file: Default file '{default_path}' doesn't exist!"
+            )
+        return default_path
+
     def set_pd_gains(self):
         # set control gains
         self._robot_entity.set_dofs_kp(

aegis_gym/rsl/envs/utils.py

@@ -145,5 +145,9 @@ def get_task_cfgs():
         "default_arm_dof": [0.0, -2.09, 2.09, -1.57, -1.57, 0.0],
         "default_gripper_dof": [0.025, 0.025],
         "ik_method": "dls_ik",
+        "urdf_model_id": {
+            "cell": "4ae9243a9e294db998d3d6e0b5a0539b",
+            "no_cell": "3b30eed8cea6423a99d9bad3343740ed",
+        },
     }
     return env_cfg, reward_scales, robot_cfg

aegis_gym/rsl/grasp_cfgs.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""
+Upload robot simulator assets to ClearML Dataset.
+Handles URDF + STL/DAE models as a versioned dataset.
+"""
+
+import argparse
+import logging
+from pathlib import Path
+from typing import Optional
+
+from clearml import Dataset
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def upload_robot_assets(
+    robot_folder_path: str,
+    dataset_name: str = "robot_simulator_assets",
+    dataset_project: str = "DeepRL",
+    parent: str = None,
+    output_storage: Optional[str] = None,  # None = use default ClearML file server
+    description: Optional[str] = None,
+):
+    """
+    Upload robot simulator folder to ClearML Dataset.
+
+    Args:
+        robot_folder_path: Path to folder containing URDF + models (STL/DAE)
+        dataset_name: Name of the dataset (version auto-incremented)
+        dataset_project: Project name in ClearML
+        output_storage: Target storage (None=default fileserver, or "/path", "s3://bucket", etc.)
+        description: Optional dataset description
+    """
+
+    folder_path = Path(robot_folder_path).resolve()
+
+    if not folder_path.exists():
+        raise FileNotFoundError(f"Folder not found: {robot_folder_path}")
+
+    logger.info(f"> Creating dataset: {dataset_project}/{dataset_name}")
+    if parent:
+        logger.info(f"> Parent DatasetID: {parent}")
+    logger.info(f"> Source folder: {folder_path}")
+    logger.info(f"> Files to upload: {len(list(folder_path.rglob('*')))}")
+
+    # Create dataset (automatically increments version if exists)
+    parent_list = [parent] if parent else None
+    dataset = Dataset.create(
+        dataset_name=dataset_name,
+        dataset_project=dataset_project,
+        parent_datasets=parent_list,
+        description=description or "Robot simulator assets",
+    )
+
+    logger.info(f"> Dataset created: {dataset.id}")
+
+    # Add all files from folder (preserves structure)
+    logger.info("> Adding files to dataset...")
+    num_files = dataset.add_files(path=str(folder_path), recursive=True, verbose=True)
+
+    logger.info(f"> Added {num_files} files")
+
+    # Upload to storage
+    logger.info("> Uploading to ClearML server...")
+    dataset.upload(
+        show_progress=True,
+        output_url=output_storage,  # Uses default if None
+    )
+
+    logger.info("> Upload complete")
+
+    # Finalize (lock version, prevents further modifications)
+    logger.info("> Finalizing dataset...")
+    dataset.finalize()
+
+    logger.info("\n> SUCCESS!")
+    logger.info(f"> Dataset ID: {dataset.id}")
+    logger.info(f"> Project: {dataset_project}")
+    logger.info(f"> Name: {dataset_name}")
+    logger.info(f"> Use in code: Dataset.get(dataset_id='{dataset.id}')")
+
+    return dataset.id
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Upload robot assets to ClearML")
+    parser.add_argument("folder", help="Path to robot model folder")
+    parser.add_argument("--name", default="AegisURDFModel", help="Dataset name")
+    parser.add_argument("--project", default="AEGIS_GRASP", help="ClearML project")
+    parser.add_argument(
+        "--parent",
+        default=None,
+        help="Already existing dataset ID which will be the parent of this version.",
+    )
+    parser.add_argument(
+        "--storage", default=None, help="Storage URL (s3://, /path, etc.)"
+    )
+    parser.add_argument("--desc", default=None, help="Dataset description")
+
+    args = parser.parse_args()
+
+    upload_robot_assets(
+        robot_folder_path=args.folder,
+        dataset_name=args.name,
+        dataset_project=args.project,
+        parent=args.parent,
+        output_storage=args.storage,
+        description=args.desc,
+    )