diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 2e9e08a9..b4c43c99 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -24,7 +24,7 @@ Guidelines for modifications: * [Zhenbiao Huang](https://github.com/legubiao) * [Zhihao Liu](https://github.com/Luo1imasi) * [Zsibot](https://www.zsibot.com) - +* [Linden713](https://github.com/linden713) ## Acknowledgements * Shiyu Zhou diff --git a/scripts/reinforcement_learning/rsl_rl/cli_args.py b/scripts/reinforcement_learning/rsl_rl/cli_args.py index d12e31e8..884c5fcc 100644 --- a/scripts/reinforcement_learning/rsl_rl/cli_args.py +++ b/scripts/reinforcement_learning/rsl_rl/cli_args.py @@ -1,10 +1,10 @@ # Copyright (c) 2024-2025 Ziqi Fan # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024-2025, The Isaac Lab Project Developers. +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). # All rights reserved. # -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations diff --git a/scripts/reinforcement_learning/rsl_rl/play.py b/scripts/reinforcement_learning/rsl_rl/play.py index 461f0cd4..fd55146f 100644 --- a/scripts/reinforcement_learning/rsl_rl/play.py +++ b/scripts/reinforcement_learning/rsl_rl/play.py @@ -1,10 +1,10 @@ # Copyright (c) 2024-2025 Ziqi Fan # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024-2025, The Isaac Lab Project Developers. +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). # All rights reserved. # -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: BSD-3-Clause """Script to play a checkpoint if an RL agent from RSL-RL.""" @@ -78,7 +78,10 @@ from isaaclab.utils.assets import retrieve_file_path from isaaclab.utils.dict import print_dict from isaaclab.utils.pretrained_checkpoint import get_published_pretrained_checkpoint + from isaaclab_rl.rsl_rl import RslRlBaseRunnerCfg, RslRlVecEnvWrapper, export_policy_as_jit, export_policy_as_onnx + +import isaaclab_tasks # noqa: F401 from isaaclab_tasks.utils import get_checkpoint_path from isaaclab_tasks.utils.hydra import hydra_task_config @@ -90,10 +93,11 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen """Play with RSL-RL agent.""" # grab task name for checkpoint path task_name = args_cli.task.split(":")[-1] + train_task_name = task_name.replace("-Play", "") # override configurations with non-hydra CLI arguments agent_cfg: RslRlBaseRunnerCfg = cli_args.update_rsl_rl_cfg(agent_cfg, args_cli) - env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else 64 + env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs # set the environment seed # note: certain randomizations occur in the environment initialization so we set the seed here @@ -134,7 +138,7 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen log_root_path = os.path.abspath(log_root_path) print(f"[INFO] Loading experiment from directory: {log_root_path}") if args_cli.use_pretrained_checkpoint: - resume_path = get_published_pretrained_checkpoint("rsl_rl", task_name) + resume_path = get_published_pretrained_checkpoint("rsl_rl", train_task_name) if not resume_path: print("[INFO] Unfortunately a pre-trained checkpoint is currently unavailable for this task.") return @@ -217,7 +221,6 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen with torch.inference_mode(): # agent stepping actions = policy(obs) - # actions = torch.zeros_like(actions) # env stepping obs, _, _, _ = env.step(actions) if args_cli.video: diff --git a/scripts/reinforcement_learning/rsl_rl/train.py b/scripts/reinforcement_learning/rsl_rl/train.py index 3e7a3d36..b9dec6d3 100644 --- a/scripts/reinforcement_learning/rsl_rl/train.py +++ b/scripts/reinforcement_learning/rsl_rl/train.py @@ -1,17 +1,16 @@ # Copyright (c) 2024-2025 Ziqi Fan # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024-2025, The Isaac Lab Project Developers. +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). # All rights reserved. # -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: BSD-3-Clause """Script to train RL agent with RSL-RL.""" """Launch Isaac Sim Simulator first.""" import argparse -import os import sys from isaaclab.app import AppLauncher @@ -55,6 +54,7 @@ """Check for minimum supported RSL-RL version.""" import importlib.metadata as metadata +import platform from packaging import version @@ -62,7 +62,10 @@ RSL_RL_VERSION = "3.0.1" installed_version = metadata.version("rsl-rl-lib") if version.parse(installed_version) < version.parse(RSL_RL_VERSION): - cmd = [r"python", "-m", "pip", "install", f"rsl-rl-lib=={RSL_RL_VERSION}"] + if platform.system() == "Windows": + cmd = [r".\isaaclab.bat", "-p", "-m", "pip", "install", f"rsl-rl-lib=={RSL_RL_VERSION}"] + else: + cmd = ["./isaaclab.sh", "-p", "-m", "pip", "install", f"rsl-rl-lib=={RSL_RL_VERSION}"] print( f"Please install the correct version of RSL-RL.\nExisting version is: '{installed_version}'" f" and required version is: '{RSL_RL_VERSION}'.\nTo install the correct version, run:" @@ -73,6 +76,7 @@ """Rest everything follows.""" import gymnasium as gym +import os import torch from datetime import datetime @@ -87,8 +91,11 @@ multi_agent_to_single_agent, ) from isaaclab.utils.dict import print_dict -from isaaclab.utils.io import dump_pickle, dump_yaml +from isaaclab.utils.io import dump_yaml + from isaaclab_rl.rsl_rl import RslRlBaseRunnerCfg, RslRlVecEnvWrapper + +import isaaclab_tasks # noqa: F401 from isaaclab_tasks.utils import get_checkpoint_path from isaaclab_tasks.utils.hydra import hydra_task_config @@ -192,8 +199,6 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen # dump the configuration into log-directory dump_yaml(os.path.join(log_dir, "params", "env.yaml"), env_cfg) dump_yaml(os.path.join(log_dir, "params", "agent.yaml"), agent_cfg) - dump_pickle(os.path.join(log_dir, "params", "env.pkl"), env_cfg) - dump_pickle(os.path.join(log_dir, "params", "agent.pkl"), agent_cfg) # run training runner.learn(num_learning_iterations=agent_cfg.max_iterations, init_at_random_ep_len=True) diff --git a/scripts/reinforcement_learning/skrl/play.py b/scripts/reinforcement_learning/skrl/play.py index fe3aaa49..45dc9cc7 100644 --- a/scripts/reinforcement_learning/skrl/play.py +++ b/scripts/reinforcement_learning/skrl/play.py @@ -1,10 +1,10 @@ # Copyright (c) 2024-2025 Ziqi Fan # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024-2025, The Isaac Lab Project Developers. +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). # All rights reserved. # -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: BSD-3-Clause """ Script to play a checkpoint of an RL agent from skrl. @@ -71,7 +71,6 @@ # clear out sys.argv for Hydra sys.argv = [sys.argv[0]] + hydra_args - # launch omniverse app app_launcher = AppLauncher(args_cli) simulation_app = app_launcher.app @@ -88,7 +87,7 @@ from packaging import version # check for minimum supported skrl version -SKRL_VERSION = "1.4.2" +SKRL_VERSION = "1.4.3" if version.parse(skrl.__version__) < version.parse(SKRL_VERSION): skrl.logger.error( f"Unsupported skrl version: {skrl.__version__}. " @@ -110,7 +109,10 @@ ) from isaaclab.utils.dict import print_dict from isaaclab.utils.pretrained_checkpoint import get_published_pretrained_checkpoint + from isaaclab_rl.skrl import SkrlVecEnvWrapper + +import isaaclab_tasks # noqa: F401 from isaaclab_tasks.utils import get_checkpoint_path from isaaclab_tasks.utils.hydra import hydra_task_config @@ -122,11 +124,16 @@ agent_cfg_entry_point = "skrl_cfg_entry_point" if algorithm in ["ppo"] else f"skrl_{algorithm}_cfg_entry_point" else: agent_cfg_entry_point = args_cli.agent + algorithm = agent_cfg_entry_point.split("_cfg")[0].split("skrl_")[-1].lower() @hydra_task_config(args_cli.task, agent_cfg_entry_point) def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, experiment_cfg: dict): """Play with skrl agent.""" + # grab task name for checkpoint path + task_name = args_cli.task.split(":")[-1] + train_task_name = task_name.replace("-Play", "") + # override configurations with non-hydra CLI arguments env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device @@ -135,7 +142,7 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, expe if args_cli.ml_framework.startswith("jax"): skrl.config.jax.backend = "jax" if args_cli.ml_framework == "jax" else "numpy" - # randomly sample a seed if seed = -1 + # randomly sample a seed if seed = -1 if args_cli.seed == -1: args_cli.seed = random.randint(0, 10000) @@ -144,15 +151,13 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, expe experiment_cfg["seed"] = args_cli.seed if args_cli.seed is not None else experiment_cfg["seed"] env_cfg.seed = experiment_cfg["seed"] - task_name = args_cli.task.split(":")[-1] - # specify directory for logging experiments (load checkpoint) log_root_path = os.path.join("logs", "skrl", experiment_cfg["agent"]["experiment"]["directory"]) log_root_path = os.path.abspath(log_root_path) print(f"[INFO] Loading experiment from directory: {log_root_path}") # get checkpoint path if args_cli.use_pretrained_checkpoint: - resume_path = get_published_pretrained_checkpoint("skrl", task_name) + resume_path = get_published_pretrained_checkpoint("skrl", train_task_name) if not resume_path: print("[INFO] Unfortunately a pre-trained checkpoint is currently unavailable for this task.") return diff --git a/scripts/reinforcement_learning/skrl/train.py b/scripts/reinforcement_learning/skrl/train.py index e19ea80a..8ef3190c 100644 --- a/scripts/reinforcement_learning/skrl/train.py +++ b/scripts/reinforcement_learning/skrl/train.py @@ -1,10 +1,10 @@ # Copyright (c) 2024-2025 Ziqi Fan # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024-2025, The Isaac Lab Project Developers. +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). # All rights reserved. # -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: BSD-3-Clause """ Script to train RL agent with skrl. @@ -107,11 +107,14 @@ ) from isaaclab.utils.assets import retrieve_file_path from isaaclab.utils.dict import print_dict -from isaaclab.utils.io import dump_pickle, dump_yaml +from isaaclab.utils.io import dump_yaml + from isaaclab_rl.skrl import SkrlVecEnvWrapper + +import isaaclab_tasks # noqa: F401 from isaaclab_tasks.utils.hydra import hydra_task_config -import robot_lab # noqa: F401 +import robot_lab # noqa: F401 # config shortcuts if args_cli.agent is None: @@ -119,6 +122,7 @@ agent_cfg_entry_point = "skrl_cfg_entry_point" if algorithm in ["ppo"] else f"skrl_{algorithm}_cfg_entry_point" else: agent_cfg_entry_point = args_cli.agent + algorithm = agent_cfg_entry_point.split("_cfg")[0].split("skrl_")[-1].lower() @hydra_task_config(args_cli.task, agent_cfg_entry_point) @@ -167,8 +171,6 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen # dump the configuration into log-directory dump_yaml(os.path.join(log_dir, "params", "env.yaml"), env_cfg) dump_yaml(os.path.join(log_dir, "params", "agent.yaml"), agent_cfg) - dump_pickle(os.path.join(log_dir, "params", "env.pkl"), env_cfg) - dump_pickle(os.path.join(log_dir, "params", "agent.pkl"), agent_cfg) # get checkpoint path (to resume training) resume_path = retrieve_file_path(args_cli.checkpoint) if args_cli.checkpoint else None diff --git a/scripts/tools/sync_training_scripts.sh b/scripts/tools/sync_training_scripts.sh new file mode 100644 index 00000000..66e6dd65 --- /dev/null +++ b/scripts/tools/sync_training_scripts.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Sync skrl and rsl_rl train/play scripts from Isaac Lab releases and reapply local tweaks. +set -euo pipefail + +fetch() { + local url="$1" + local dest="$2" + echo "Fetching ${url} to ${dest}" + curl -fsSL "${url}" -o "${dest}" +} + +# Sync for skrl +SKRL_BASE_URL="https://raw.githubusercontent.com/isaac-sim/IsaacLab/main/scripts/reinforcement_learning/skrl" +SKRL_DEST_DIR="scripts/reinforcement_learning/skrl" +SKRL_FILES=(train.py play.py) + +echo "Syncing skrl scripts..." +for name in "${SKRL_FILES[@]}"; do + url="${SKRL_BASE_URL}/${name}" + dest_path="${SKRL_DEST_DIR}/${name}" + fetch "${url}" "${dest_path}" +done + +# Sync for rsl_rl +RSL_RL_BASE_URL="https://raw.githubusercontent.com/isaac-sim/IsaacLab/main/scripts/reinforcement_learning/rsl_rl" +RSL_RL_DEST_DIR="scripts/reinforcement_learning/rsl_rl" +RSL_RL_FILES=(train.py play.py cli_args.py) + +echo "Syncing rsl_rl scripts..." +for name in "${RSL_RL_FILES[@]}"; do + url="${RSL_RL_BASE_URL}/${name}" + dest_path="${RSL_RL_DEST_DIR}/${name}" + fetch "${url}" "${dest_path}" +done + +echo "Done."