[NEWTON]Enables omni-client be installed with packman

isaaclab.sh

@@ -348,6 +348,8 @@ while [[ $# -gt 0 ]]; do
             # install the python packages in IsaacLab/source directory
             echo "[INFO] Installing extensions inside the Isaac Lab repository..."
             python_exe=$(extract_python_exe)
+            # install omni.client via packman helper
+            ${python_exe} "${ISAACLAB_PATH}/tools/installation/install_omni_client_packman.py"
             # check if pytorch is installed and its version
             # install pytorch with cuda 12.8 for blackwell support
             if ${python_exe} -m pip list 2>/dev/null | grep -q "torch"; then

source/isaaclab/isaaclab/sim/spawners/from_files/from_files.py

@@ -28,6 +28,7 @@
     select_usd_variants,
 )
 from isaaclab.sim.utils.stage import get_current_stage
+from isaaclab.utils.assets import check_file_path, retrieve_file_path
 
 if TYPE_CHECKING:
     from . import from_files_cfg
@@ -258,20 +259,16 @@ def _spawn_from_usd_file(
     Raises:
         FileNotFoundError: If the USD file does not exist at the given path.
     """
-    # get stage handle
-    stage = get_current_stage()
-
-    # check file path exists
-    if not stage.ResolveIdentifierToEditTarget(usd_path):
-        if "4.5" in usd_path:
-            usd_5_0_path = (
-                usd_path.replace("http", "https").replace("-production.", "-staging.").replace("/4.5", "/5.0")
-            )
-            if not stage.ResolveIdentifierToEditTarget(usd_5_0_path):
-                raise FileNotFoundError(f"USD file not found at path at either: '{usd_path}' or '{usd_5_0_path}'.")
-            usd_path = usd_5_0_path
-        else:
-            raise FileNotFoundError(f"USD file not found at path at: '{usd_path}'.")
+    # check file path exists (supports local paths, S3, HTTP/HTTPS URLs)
+    # check_file_path returns: 0 (not found), 1 (local), 2 (remote)
+    file_status = check_file_path(usd_path)
+    if file_status == 0:
+        raise FileNotFoundError(f"USD file not found at path: '{usd_path}'.")
+
+    # Download remote files (S3, HTTP, HTTPS) to local cache
+    # This also downloads all USD dependencies to maintain references
+    if file_status == 2:
+        usd_path = retrieve_file_path(usd_path)
     # spawn asset if it doesn't exist.
     if not prim_utils.is_prim_path_valid(prim_path):
         # add prim as reference to stage

source/isaaclab/isaaclab/utils/assets.py

@@ -14,15 +14,21 @@
 """
 
 import io
+import logging
 import os
+import posixpath
 import tempfile
+from pathlib import Path
 from typing import Literal
+from urllib.parse import urlparse
 
-import carb
 import omni.client
 
-NUCLEUS_ASSET_ROOT_DIR = carb.settings.get_settings().get("/persistent/isaac/asset_root/cloud")
-"""Path to the root directory on the Nucleus Server."""
+logger = logging.getLogger(__name__)
+from pxr import Sdf
+
+NUCLEUS_ASSET_ROOT_DIR = "https://omniverse-content-production.s3-us-west-2.amazonaws.com/Assets/Isaac/5.0"
+"""Path to the root directory on the cloud storage."""
 
 NVIDIA_NUCLEUS_DIR = f"{NUCLEUS_ASSET_ROOT_DIR}/NVIDIA"
 """Path to the root directory on the NVIDIA Nucleus Server."""
@@ -33,6 +39,8 @@
 ISAACLAB_NUCLEUS_DIR = f"{ISAAC_NUCLEUS_DIR}/IsaacLab"
 """Path to the ``Isaac/IsaacLab`` directory on the NVIDIA Nucleus Server."""
 
+USD_EXTENSIONS = {".usd", ".usda", ".usdz"}
+
 
 def check_file_path(path: str) -> Literal[0, 1, 2]:
     """Checks if a file exists on the Nucleus Server or locally.
@@ -91,16 +99,38 @@ def retrieve_file_path(path: str, download_dir: str | None = None, force_downloa
         # create download directory if it does not exist
         if not os.path.exists(download_dir):
             os.makedirs(download_dir)
-        # download file in temp directory using os
-        file_name = os.path.basename(omni.client.break_url(path.replace(os.sep, "/")).path)
-        target_path = os.path.join(download_dir, file_name)
-        # check if file already exists locally
-        if not os.path.isfile(target_path) or force_download:
-            # copy file to local machine
-            result = omni.client.copy(path.replace(os.sep, "/"), target_path, omni.client.CopyBehavior.OVERWRITE)
-            if result != omni.client.Result.OK and force_download:
-                raise RuntimeError(f"Unable to copy file: '{path}'. Is the Nucleus Server running?")
-        return os.path.abspath(target_path)
+        # recursive download: mirror remote tree under download_dir
+        remote_url = path.replace(os.sep, "/")
+        to_visit = [remote_url]
+        visited = set()
+        local_root = None
+
+        while to_visit:
+            cur_url = to_visit.pop()
+            if cur_url in visited:
+                continue
+            visited.add(cur_url)
+
+            cur_rel = urlparse(cur_url).path.lstrip("/")
+            target_path = os.path.join(download_dir, cur_rel)
+            os.makedirs(os.path.dirname(target_path), exist_ok=True)
+
+            if not os.path.isfile(target_path) or force_download:
+                result = omni.client.copy(cur_url, target_path, omni.client.CopyBehavior.OVERWRITE)
+                if result != omni.client.Result.OK and force_download:
+                    raise RuntimeError(f"Unable to copy file: '{cur_url}'. Is the Nucleus Server running?")
+
+            if local_root is None:
+                local_root = target_path
+
+            # recurse into USD dependencies and referenced assets
+            if Path(target_path).suffix.lower() in USD_EXTENSIONS:
+                for ref in _find_usd_references(target_path):
+                    ref_url = _resolve_reference_url(cur_url, ref)
+                    if ref_url and ref_url not in visited:
+                        to_visit.append(ref_url)
+
+        return os.path.abspath(local_root)
     else:
         raise FileNotFoundError(f"Unable to find the file: {path}")
 
@@ -127,3 +157,102 @@ def read_file(path: str) -> io.BytesIO:
         return io.BytesIO(memoryview(file_content).tobytes())
     else:
         raise FileNotFoundError(f"Unable to find the file: {path}")
+
+
+def _is_downloadable_asset(path: str) -> bool:
+    """Return True for USD or other asset types we mirror locally (textures, etc.)."""
+    clean = path.split("?", 1)[0].split("#", 1)[0]
+    suffix = Path(clean).suffix.lower()
+
+    if suffix == ".mdl":
+        # MDL modules (OmniPBR.mdl, OmniSurface.mdl, ...) come from MDL search paths
+        return False
+    if not suffix:
+        return False
+    if suffix not in {".usd", ".usda", ".usdz", ".png", ".jpg", ".jpeg", ".exr", ".hdr", ".tif", ".tiff"}:
+        return False
+    return True
+
+
+def _find_usd_references(local_usd_path: str) -> set[str]:
+    """Use Sdf API to collect referenced assets from a USD layer."""
+    try:
+        layer = Sdf.Layer.FindOrOpen(local_usd_path)
+    except Exception:
+        logger.warning("Failed to open USD layer: %s", local_usd_path, exc_info=True)
+        return set()
+
+    if layer is None:
+        return set()
+
+    refs: set[str] = set()
+
+    # Sublayers
+    for sub_path in getattr(layer, "subLayerPaths", []) or []:
+        if sub_path and _is_downloadable_asset(sub_path):
+            refs.add(str(sub_path))
+
+    def _walk_prim(prim_spec: Sdf.PrimSpec) -> None:
+        # References
+        ref_list = prim_spec.referenceList
+        for field in ("addedItems", "prependedItems", "appendedItems", "explicitItems"):
+            items = getattr(ref_list, field, None)
+            if not items:
+                continue
+            for ref in items:
+                asset_path = getattr(ref, "assetPath", None)
+                if asset_path and _is_downloadable_asset(asset_path):
+                    refs.add(str(asset_path))
+
+        # Payloads
+        payload_list = prim_spec.payloadList
+        for field in ("addedItems", "prependedItems", "appendedItems", "explicitItems"):
+            items = getattr(payload_list, field, None)
+            if not items:
+                continue
+            for payload in items:
+                asset_path = getattr(payload, "assetPath", None)
+                if asset_path and _is_downloadable_asset(asset_path):
+                    refs.add(str(asset_path))
+
+        # AssetPath-valued attributes (this is where OmniPBR.mdl, textures, etc. show up)
+        for attr_spec in prim_spec.attributes.values():
+            default = attr_spec.default
+            if isinstance(default, Sdf.AssetPath):
+                if default.path and _is_downloadable_asset(default.path):
+                    refs.add(default.path)
+            elif isinstance(default, Sdf.AssetPathArray):
+                for ap in default:
+                    if ap.path and _is_downloadable_asset(ap.path):
+                        refs.add(ap.path)
+
+        for child in prim_spec.nameChildren.values():
+            _walk_prim(child)
+
+    for root_prim in layer.rootPrims.values():
+        _walk_prim(root_prim)
+
+    return refs
+
+
+def _resolve_reference_url(base_url: str, ref: str) -> str:
+    """Resolve a USD asset reference against a base URL (http/local)."""
+    ref = ref.strip()
+    if not ref:
+        return ref
+
+    parsed_ref = urlparse(ref)
+    if parsed_ref.scheme:
+        return ref
+
+    base = urlparse(base_url)
+    if base.scheme == "":
+        base_dir = os.path.dirname(base_url)
+        return os.path.normpath(os.path.join(base_dir, ref))
+
+    base_dir = posixpath.dirname(base.path)
+    if ref.startswith("/"):
+        new_path = posixpath.normpath(ref)
+    else:
+        new_path = posixpath.normpath(posixpath.join(base_dir, ref))
+    return f"{base.scheme}://{base.netloc}{new_path}"

tools/installation/install_omni_client_packman.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""
+Install omni.client from a prebuilt 7z payload into the current Python environment.
+
+- Downloads https://d4i3qtqj3r0z5.cloudfront.net/omni_client_library.linux-x86_64@<version>.7z
+- Extracts to a cache dir (default: $TMPDIR/omni_client_cache; override with OMNI_CLIENT_CACHE)
+- Copies the Python package (release/bindings-python) and native libs (release/*.so) into site-packages/_omni_client
+- Drops a .pth for import visibility
+- Creates a minimal dist-info so `pip uninstall omni-client-offline` works
+
+# TODO: Once pip has been shipped, remove this script and use pip install omniverseclient==<version> instead.
+"""
+
+import logging
+import os
+import pathlib
+import shutil
+import site
+import subprocess
+import sys
+import tempfile
+import urllib.request
+
+# Ensure py7zr is available
+try:
+    import py7zr  # type: ignore  # noqa: F401
+except ImportError:
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "py7zr"])
+    import py7zr  # type: ignore
+
+
+logger = logging.getLogger(__name__)
+
+# Configuration
+pkg_ver = os.environ.get("OMNI_CLIENT_VERSION", "2.68.1")
+cache_root = pathlib.Path(os.environ.get("OMNI_CLIENT_CACHE", tempfile.gettempdir())) / "omni_client_cache"
+payload_url = f"https://d4i3qtqj3r0z5.cloudfront.net/omni_client_library.linux-x86_64%40{pkg_ver}.7z"
+
+# Paths
+cache_root.mkdir(parents=True, exist_ok=True)
+payload = cache_root / f"omni_client.{pkg_ver}.7z"
+extract_root = cache_root / f"omni_client.{pkg_ver}.extracted"
+
+# Download payload if missing
+if not payload.exists():
+    logger.info(f" Downloading omni.client payload from {payload_url} ...")
+    urllib.request.urlretrieve(payload_url, payload)
+
+# Extract payload only if not already present
+extract_root.mkdir(parents=True, exist_ok=True)
+already_extracted = (extract_root / "release" / "bindings-python" / "omni" / "client").exists()
+if not already_extracted:
+    logger.info(f" Extracting omni.client payload into {extract_root} ...")
+    with py7zr.SevenZipFile(payload, mode="r") as z:
+        z.extractall(path=extract_root)
+else:
+    logger.info(f" Reusing existing extraction at {extract_root}")
+
+# Locate python package and native libs
+src_py = extract_root / "release" / "bindings-python"
+if not (src_py / "omni" / "client").exists():
+    raise RuntimeError(f"Could not locate omni.client python package at {src_py}")
+
+src_lib = extract_root / "release"
+if not any(src_lib.glob("libomni*.so*")):
+    raise RuntimeError(f"Could not locate native libs under {src_lib}")
+
+# Install into site-packages
+if hasattr(site, "getsitepackages"):
+    candidates = [pathlib.Path(p) for p in site.getsitepackages() if p.startswith(sys.prefix)]
+    site_pkgs = candidates[0] if candidates else pathlib.Path(site.getusersitepackages())
+else:
+    site_pkgs = pathlib.Path(site.getusersitepackages())
+dest = site_pkgs / "_omni_client"
+dest.mkdir(parents=True, exist_ok=True)
+shutil.copytree(src_py, dest, dirs_exist_ok=True)
+shutil.copytree(src_lib, dest / "lib", dirs_exist_ok=True)
+
+# Ensure the extension can find its libs without env vars
+client_dir = dest / "omni" / "client"
+client_dir.mkdir(parents=True, exist_ok=True)
+for libfile in (dest / "lib").glob("libomni*.so*"):
+    target = client_dir / libfile.name
+    if not target.exists():
+        try:
+            target.symlink_to(libfile)
+        except Exception:
+            shutil.copy2(libfile, target)
+
+# Add .pth for import visibility
+with open(site_pkgs / "omni_client.pth", "w", encoding="utf-8") as f:
+    f.write(str(dest) + "\n")
+    f.write(str(dest / "lib") + "\n")
+
+# Minimal dist-info so pip can uninstall (pip uninstall omni-client)
+dist_name = "omni-client"
+dist_info = site_pkgs / f"{dist_name.replace('-', '_')}-{pkg_ver}.dist-info"
+dist_info.mkdir(parents=True, exist_ok=True)
+(dist_info / "INSTALLER").write_text("manual\n", encoding="utf-8")
+metadata = "\n".join([
+    f"Name: {dist_name}",
+    f"Version: {pkg_ver}",
+    "Summary: Offline omni.client bundle",
+    "",
+])
+(dist_info / "METADATA").write_text(metadata, encoding="utf-8")
+
+records = []
+for path in [
+    site_pkgs / "omni_client.pth",
+    dist_info / "INSTALLER",
+    dist_info / "METADATA",
+]:
+    records.append(f"{path.relative_to(site_pkgs)},,")
+for path in dest.rglob("*"):
+    records.append(f"{path.relative_to(site_pkgs)},,")
+for path in dist_info.rglob("*"):
+    if path.name != "RECORD":
+        records.append(f"{path.relative_to(site_pkgs)},,")
+(dist_info / "RECORD").write_text("\n".join(records), encoding="utf-8")
+
+logger.info(f"Installed omni.client to {dest} (dist: {dist_info.name})")
+logger.info("Uninstall with: pip uninstall omni-client")