diff --git a/source/isaaclab_tasks/test/benchmarking/configs.yaml b/source/isaaclab_tasks/test/benchmarking/configs.yaml index b4bc4fc043f..5514a91d668 100644 --- a/source/isaaclab_tasks/test/benchmarking/configs.yaml +++ b/source/isaaclab_tasks/test/benchmarking/configs.yaml @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause + # mode for very simple functional testing without checking thresholds fast_test: rl_games:Isaac-Ant-v0: @@ -16,7 +17,7 @@ fast_test: # mode for capturing KPIs across all environments without checking thresholds full_test: Isaac-*: - max_iterations: 500 + max_iterations: 10 lower_thresholds: reward: -99999 episode_length: -99999 @@ -67,9 +68,31 @@ fast: episode_length: 900 upper_thresholds: duration: 1800 + sb3:Isaac-Lift-Cube-Franka-v0: + max_iterations: 100 + lower_thresholds: + reward: 50 + episode_length: 80 + upper_thresholds: + duration: 800 + rl_games:Isaac-Dexsuite-Kuka-Allegro-Lift-v0: + max_iterations: 100 + lower_thresholds: + reward: 30 + episode_length: 150 + upper_thresholds: + duration: 800 + rsl_rl:Isaac-Reach-OpenArm-v0: + max_iterations: 200 + lower_thresholds: + reward: 0.15 + episode_length: 100 + upper_thresholds: + duration: 600 -# mode for weekly CI +# mode for nightly CI full: + # Locomotion tasks - basic Isaac-Ant-Direct-v0: max_iterations: 300 lower_thresholds: @@ -84,6 +107,21 @@ full: episode_length: 700 upper_thresholds: duration: 800 + Isaac-Humanoid-Direct-v0: + max_iterations: 300 + lower_thresholds: + reward: 2000 + episode_length: 600 + upper_thresholds: + duration: 1000 + Isaac-Humanoid-v0: + max_iterations: 1000 + lower_thresholds: + reward: 100 + episode_length: 600 + upper_thresholds: + duration: 2500 + # Classic control tasks Isaac-Cart-Double-Pendulum-Direct-v0: max_iterations: 300 lower_thresholds: @@ -91,27 +129,35 @@ full: episode_length: 150 upper_thresholds: duration: 500 - Isaac-Cartpole-Depth-Camera-Direct-v0: + Isaac-Cartpole-Direct-v0: max_iterations: 300 lower_thresholds: reward: 200 episode_length: 150 upper_thresholds: - duration: 3000 - Isaac-Cartpole-Depth-v0: + duration: 500 + Isaac-Cartpole-v0: + max_iterations: 1000 + lower_thresholds: + reward: 3 + episode_length: 150 + upper_thresholds: + duration: 1500 + # Vision-based cartpole tasks + Isaac-Cartpole-Depth-Camera-Direct-v0: max_iterations: 300 lower_thresholds: - reward: 1 + reward: 200 episode_length: 150 upper_thresholds: duration: 3000 - Isaac-Cartpole-Direct-v0: + Isaac-Cartpole-Depth-v0: max_iterations: 300 lower_thresholds: - reward: 200 + reward: 1 episode_length: 150 upper_thresholds: - duration: 500 + duration: 3000 Isaac-Cartpole-RGB-Camera-Direct-v0: max_iterations: 300 lower_thresholds: @@ -140,34 +186,7 @@ full: episode_length: 150 upper_thresholds: duration: 4000 - Isaac-Cartpole-v0: - max_iterations: 1000 - lower_thresholds: - reward: 3 - episode_length: 150 - upper_thresholds: - duration: 1500 - Isaac-Factory-GearMesh-Direct-v0: - max_iterations: 100 - lower_thresholds: - reward: 200 - episode_length: 250 - upper_thresholds: - duration: 6000 - Isaac-Factory-NutThread-Direct-v0: - max_iterations: 100 - lower_thresholds: - reward: 400 - episode_length: 400 - upper_thresholds: - duration: 5000 - Isaac-Factory-PegInsert-Direct-v0: - max_iterations: 100 - lower_thresholds: - reward: 125 - episode_length: 130 - upper_thresholds: - duration: 4000 + # Manipulation tasks - Franka Isaac-Franka-Cabinet-Direct-v0: max_iterations: 300 lower_thresholds: @@ -175,55 +194,64 @@ full: episode_length: 400 upper_thresholds: duration: 1000 - Isaac-Humanoid-Direct-v0: + Isaac-Lift-Cube-Franka-v0: max_iterations: 300 lower_thresholds: - reward: 2000 - episode_length: 600 + reward: 90 + episode_length: 100 upper_thresholds: duration: 1000 - Isaac-Humanoid-v0: + Isaac-Open-Drawer-Franka-v0: + max_iterations: 200 + lower_thresholds: + reward: 60 + episode_length: 150 + upper_thresholds: + duration: 3000 + Isaac-Reach-Franka-*: max_iterations: 1000 lower_thresholds: - reward: 100 - episode_length: 600 + reward: 0.25 + episode_length: 150 upper_thresholds: - duration: 2500 - Isaac-Lift-Cube-Franka-v0: - max_iterations: 300 + duration: 1500 + Isaac-Reach-Franka-OSC-v0: + max_iterations: 1000 lower_thresholds: - reward: 90 - episode_length: 100 + reward: 0.25 + episode_length: 150 upper_thresholds: - duration: 1000 - Isaac-Navigation-Flat-Anymal-C-v0: + duration: 1500 + # Manipulation tasks - OpenArm + Isaac-Lift-Cube-OpenArm-v0: max_iterations: 300 lower_thresholds: - reward: 0.5 - episode_length: 20 + reward: 80 + episode_length: 100 upper_thresholds: - duration: 2000 - Isaac-Open-Drawer-Franka-v0: + duration: 1200 + Isaac-Open-Drawer-OpenArm-v0: max_iterations: 200 lower_thresholds: - reward: 60 + reward: 50 episode_length: 150 upper_thresholds: duration: 3000 - Isaac-Quadcopter-Direct-v0: - max_iterations: 500 + Isaac-Reach-OpenArm-v0: + max_iterations: 1000 lower_thresholds: - reward: 90 - episode_length: 300 + reward: 0.25 + episode_length: 150 upper_thresholds: - duration: 500 - Isaac-Reach-Franka-*: + duration: 1500 + Isaac-Reach-OpenArm-Bi-v0: max_iterations: 1000 lower_thresholds: reward: 0.25 episode_length: 150 upper_thresholds: - duration: 1500 + duration: 1800 + # Manipulation tasks - UR10 Isaac-Reach-UR10-v0: max_iterations: 1000 lower_thresholds: @@ -231,6 +259,7 @@ full: episode_length: 150 upper_thresholds: duration: 1500 + # Dexterous manipulation - Allegro hand Isaac-Repose-Cube-Allegro-Direct-v0: max_iterations: 500 lower_thresholds: @@ -245,6 +274,7 @@ full: episode_length: 300 upper_thresholds: duration: 1500 + # Dexterous manipulation - Shadow hand Isaac-Repose-Cube-Shadow-Direct-v0: max_iterations: 3000 lower_thresholds: @@ -280,6 +310,87 @@ full: episode_length: 150 upper_thresholds: duration: 10000 + # Dexterous manipulation - Kuka-Allegro + Isaac-Dexsuite-Kuka-Allegro-Lift-v0: + max_iterations: 500 + lower_thresholds: + reward: 50 + episode_length: 200 + upper_thresholds: + duration: 2000 + Isaac-Dexsuite-Kuka-Allegro-Reorient-v0: + max_iterations: 500 + lower_thresholds: + reward: 50 + episode_length: 200 + upper_thresholds: + duration: 2000 + # Factory and forge tasks + Isaac-AutoMate-Assembly-Direct-v0: + max_iterations: 300 + lower_thresholds: + reward: 100 + episode_length: 200 + upper_thresholds: + duration: 1500 + Isaac-AutoMate-Disassembly-Direct-v0: + max_iterations: 300 + lower_thresholds: + reward: 100 + episode_length: 200 + upper_thresholds: + duration: 1500 + Isaac-Factory-GearMesh-Direct-v0: + max_iterations: 100 + lower_thresholds: + reward: 200 + episode_length: 250 + upper_thresholds: + duration: 6000 + Isaac-Factory-NutThread-Direct-v0: + max_iterations: 100 + lower_thresholds: + reward: 400 + episode_length: 400 + upper_thresholds: + duration: 5000 + Isaac-Factory-PegInsert-Direct-v0: + max_iterations: 100 + lower_thresholds: + reward: 125 + episode_length: 130 + upper_thresholds: + duration: 4000 + Isaac-Forge-GearMesh-Direct-v0: + max_iterations: 100 + lower_thresholds: + reward: 200 + episode_length: 250 + upper_thresholds: + duration: 6000 + Isaac-Forge-NutThread-Direct-v0: + max_iterations: 100 + lower_thresholds: + reward: 400 + episode_length: 400 + upper_thresholds: + duration: 5000 + Isaac-Forge-PegInsert-Direct-v0: + max_iterations: 100 + lower_thresholds: + reward: 125 + episode_length: 130 + upper_thresholds: + duration: 4000 + # Aerial tasks + Isaac-Quadcopter-Direct-v0: + max_iterations: 500 + lower_thresholds: + reward: 90 + episode_length: 300 + upper_thresholds: + duration: 500 + # Quadruped and humanoid locomotion - flat terrain Isaac-Velocity-Flat-*: max_iterations: 1000 lower_thresholds: @@ -294,6 +405,35 @@ full: episode_length: 700 upper_thresholds: duration: 6000 + Isaac-Velocity-Flat-Digit-v0: + max_iterations: 1000 + lower_thresholds: + reward: 10 + episode_length: 700 + upper_thresholds: + duration: 3000 + Isaac-Velocity-Flat-G1-v0: + max_iterations: 1000 + lower_thresholds: + reward: 10 + episode_length: 700 + upper_thresholds: + duration: 3500 + Isaac-Velocity-Flat-Unitree-Go1-v0: + max_iterations: 1000 + lower_thresholds: + reward: 15 + episode_length: 700 + upper_thresholds: + duration: 3000 + Isaac-Velocity-Flat-Unitree-Go2-v0: + max_iterations: 1000 + lower_thresholds: + reward: 15 + episode_length: 700 + upper_thresholds: + duration: 3000 + # Quadruped and humanoid locomotion - rough terrain Isaac-Velocity-Rough-*: max_iterations: 1000 lower_thresholds: @@ -301,3 +441,39 @@ full: episode_length: 700 upper_thresholds: duration: 6000 + Isaac-Velocity-Rough-Digit-v0: + max_iterations: 1000 + lower_thresholds: + reward: 5 + episode_length: 700 + upper_thresholds: + duration: 6000 + Isaac-Velocity-Rough-G1-v0: + max_iterations: 1000 + lower_thresholds: + reward: 5 + episode_length: 700 + upper_thresholds: + duration: 6000 + Isaac-Velocity-Rough-Unitree-Go1-v0: + max_iterations: 1000 + lower_thresholds: + reward: 7 + episode_length: 700 + upper_thresholds: + duration: 6000 + Isaac-Velocity-Rough-Unitree-Go2-v0: + max_iterations: 1000 + lower_thresholds: + reward: 7 + episode_length: 700 + upper_thresholds: + duration: 6000 + # Locomotion-manipulation + Isaac-Tracking-LocoManip-Digit-v0: + max_iterations: 500 + lower_thresholds: + reward: 5 + episode_length: 500 + upper_thresholds: + duration: 3000 diff --git a/source/isaaclab_tasks/test/benchmarking/conftest.py b/source/isaaclab_tasks/test/benchmarking/conftest.py index c878d7e8823..d8b26c3d6b8 100644 --- a/source/isaaclab_tasks/test/benchmarking/conftest.py +++ b/source/isaaclab_tasks/test/benchmarking/conftest.py @@ -4,12 +4,15 @@ # SPDX-License-Identifier: BSD-3-Clause import json +from datetime import datetime import env_benchmark_test_utils as utils import pytest # Global variable for storing KPI data GLOBAL_KPI_STORE = {} +# Global variable for storing the start timestamp +START_TIMESTAMP = None def pytest_addoption(parser): @@ -91,11 +94,17 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("workflow", workflows) +# The pytest session start hook to capture the start timestamp +def pytest_sessionstart(session): + global START_TIMESTAMP + START_TIMESTAMP = datetime.now().isoformat() + + # The pytest session finish hook def pytest_sessionfinish(session, exitstatus): # Access global variable instead of fixture tag = session.config.getoption("--tag") - utils.process_kpi_data(GLOBAL_KPI_STORE, tag=tag) + utils.process_kpi_data(GLOBAL_KPI_STORE, tag=tag, timestamp=START_TIMESTAMP) print(json.dumps(GLOBAL_KPI_STORE, indent=2)) save_kpi_payload = session.config.getoption("--save_kpi_payload") if save_kpi_payload: diff --git a/source/isaaclab_tasks/test/benchmarking/env_benchmark_test_utils.py b/source/isaaclab_tasks/test/benchmarking/env_benchmark_test_utils.py index 0c939ca0166..1434f03f2b2 100644 --- a/source/isaaclab_tasks/test/benchmarking/env_benchmark_test_utils.py +++ b/source/isaaclab_tasks/test/benchmarking/env_benchmark_test_utils.py @@ -12,10 +12,27 @@ import yaml from datetime import datetime -import carb from tensorboard.backend.event_processing import event_accumulator +def _get_repo_path(): + """Get the repository root by searching for marker files. + + Searches upward from the current file for IsaacLab repository markers + (isaaclab.sh or setup.py) to robustly find the repo root. + """ + current = os.path.abspath(__file__) + # Look for isaaclab.sh or setup.py as markers (max 10 levels up) + for _ in range(10): + current = os.path.dirname(current) + if os.path.exists(os.path.join(current, "isaaclab.sh")): + return current + # Fallback marker + if os.path.exists(os.path.join(current, "setup.py")) and os.path.exists(os.path.join(current, "source")): + return current + raise RuntimeError("Could not find IsaacLab repository root. Expected to find 'isaaclab.sh' in parent directories.") + + def get_env_configs(configs_path): """Get environment configurations from yaml filepath.""" with open(configs_path) as env_configs_file: @@ -76,14 +93,17 @@ def evaluate_job(workflow, task, env_config, duration): if val is None or not isinstance(val, (int, float)) or (isinstance(val, float) and math.isnan(val)): continue val = round(val, 4) + threshold_val_rounded = round(threshold_val, 4) if uses_lower_threshold: - # print(f"{threshold_name}: {val} > {round(threshold_val, 4)}") if val < threshold_val: kpi_payload["success"] = False + if not kpi_payload["msg"]: + kpi_payload["msg"] = f"{threshold_name} below threshold: {val} < {threshold_val_rounded}" else: - # print(f"{threshold_name}: {val} < {round(threshold_val, 4)}") if val > threshold_val: kpi_payload["success"] = False + if not kpi_payload["msg"]: + kpi_payload["msg"] = f"{threshold_name} above threshold: {val} > {threshold_val_rounded}" kpi_payload[threshold_name] = val if threshold_name == "reward": normalized_reward = val / threshold_val @@ -98,8 +118,14 @@ def evaluate_job(workflow, task, env_config, duration): return kpi_payload -def process_kpi_data(kpi_payloads, tag=""): - """Combine and augment the KPI payloads.""" +def process_kpi_data(kpi_payloads, tag, timestamp): + """Combine and augment the KPI payloads. + + Args: + kpi_payloads: Dictionary of KPI payloads for each job. + tag: Tag for the KPI payload. + timestamp: Timestamp to use (ISO format). + """ # accumulate workflow outcomes totals = {} successes = {} @@ -126,7 +152,7 @@ def process_kpi_data(kpi_payloads, tag=""): "successes": successes, "failures_did_not_finish": failures_did_not_finish, "failures_did_not_pass_thresholds": failures_did_not_pass_thresholds, - "timestamp": datetime.now().isoformat(), + "timestamp": timestamp, "tag": tag, } @@ -136,7 +162,7 @@ def process_kpi_data(kpi_payloads, tag=""): def output_payloads(payloads): """Output the KPI payloads to a json file.""" # first grab all log files - repo_path = os.path.join(carb.tokens.get_tokens_interface().resolve("${app}"), "..") + repo_path = _get_repo_path() output_path = os.path.join(repo_path, "logs/kpi.json") # create directory if it doesn't exist if not os.path.exists(os.path.dirname(output_path)): @@ -149,13 +175,15 @@ def output_payloads(payloads): def _retrieve_logs(workflow, task): """Retrieve training logs.""" # first grab all log files - repo_path = os.path.join(carb.tokens.get_tokens_interface().resolve("${app}"), "..") + repo_path = _get_repo_path() from isaacsim.core.version import get_version if int(get_version()[2]) < 5: repo_path = os.path.join(repo_path, "..") if workflow == "rl_games": log_files_path = os.path.join(repo_path, f"logs/{workflow}/{task}/*/summaries/*") + elif workflow == "sb3": + log_files_path = os.path.join(repo_path, f"logs/{workflow}/{task}/*/*/*.tfevents.*") else: log_files_path = os.path.join(repo_path, f"logs/{workflow}/{task}/*/*.tfevents.*") log_files = glob.glob(log_files_path) @@ -166,6 +194,12 @@ def _retrieve_logs(workflow, task): latest_log_file = max(log_files, key=os.path.getctime) # parse tf file into a dictionary log_data = _parse_tf_logs(latest_log_file) + + # validate that log data contains entries + if not log_data: + print(f"Warning: Log file {latest_log_file} parsed but contains no data") + return None + return log_data @@ -189,7 +223,7 @@ def _extract_log_val(name, log_data, uses_lower_threshold, workflow): reward_tags = { "rl_games": "rewards/iter", "rsl_rl": "Train/mean_reward", - "sb3": None, # TODO: complete when sb3 is fixed + "sb3": "rollout/ep_rew_mean", "skrl": "Reward / Total reward (mean)", } tag = reward_tags.get(workflow) @@ -200,18 +234,17 @@ def _extract_log_val(name, log_data, uses_lower_threshold, workflow): episode_tags = { "rl_games": "episode_lengths/iter", "rsl_rl": "Train/mean_episode_length", - "sb3": None, # TODO: complete when sb3 is fixed + "sb3": "rollout/ep_len_mean", "skrl": "Episode / Total timesteps (mean)", } tag = episode_tags.get(workflow) if tag: return _extract_feature(log_data, tag, uses_lower_threshold) - - elif name == "training_time": - return {"rl_games": log_data["rewards/time"][-1][0], "rsl_rl": None, "sb3": None, "skrl": None}.get( - workflow - ) - except Exception: + except KeyError as e: + print(f"Warning: Metric '{name}' not found in logs for workflow '{workflow}': {e}") + return None + except Exception as e: + print(f"Error extracting '{name}' for workflow '{workflow}': {e}") return None raise ValueError(f"Env Config name {name} is not supported.") diff --git a/source/isaaclab_tasks/test/benchmarking/test_environments_training.py b/source/isaaclab_tasks/test/benchmarking/test_environments_training.py index 5fae937ef84..9fc625102de 100644 --- a/source/isaaclab_tasks/test/benchmarking/test_environments_training.py +++ b/source/isaaclab_tasks/test/benchmarking/test_environments_training.py @@ -64,14 +64,21 @@ def train_job(workflow, task, env_config, num_gpus): cmd.append("--distributed") # Add experiment name variable - cmd.append(f"{WORKFLOW_EXPERIMENT_NAME_VARIABLE[workflow]}={task}") + workflow_experiment_name_variable = WORKFLOW_EXPERIMENT_NAME_VARIABLE.get(workflow) + if workflow_experiment_name_variable: + cmd.append(f"{workflow_experiment_name_variable}={task}") print("Running : " + " ".join(cmd)) start_time = time.time() - subprocess.run(cmd) + result = subprocess.run(cmd, capture_output=True, text=True) duration = time.time() - start_time + if result.returncode != 0: + print(f"Training failed with exit code {result.returncode}") + print(f"STDERR: {result.stderr}") + # Still return duration so evaluate_job can report failure via logs + return duration