Media Libraries - rocDecode & rocJPEG tests

.gitmodules

@@ -68,3 +68,11 @@
 	path = math-libs/libhipcxx
 	url = https://github.com/ROCm/libhipcxx.git
 	branch = amd-develop
+[submodule "rocdecode"]
+	path = media-libs/rocdecode
+	url = https://github.com/ROCm/rocDecode.git
+	branch = therock
+[submodule "rocjpeg"]
+	path = media-libs/rocjpeg
+	url = https://github.com/ROCm/rocjpeg.git
+	branch = therock

BUILD_TOPOLOGY.toml

@@ -119,6 +119,11 @@ disable_platforms = ["windows"]
 description = "IREE integration submodules"
 submodules = ["iree", "fusilli"]
 
+[source_sets.media-libs]
+description = "ROCm media libraries submodules"
+submodules = ["rocdecode", "rocjpeg"]
+disable_platforms = ["windows"]
+
 # ==============================================================================
 # BUILD STAGES - Pipeline jobs that build sets of artifact groups
 # ==============================================================================
@@ -173,9 +178,9 @@ artifact_groups = ["iree-libs"]
 description = "Profiler applications (depends on profiler-core)"
 artifact_groups = ["profiler-apps"]
 
-[build_stages.media]
+[build_stages.media-libs]
 description = "Media Libraries"
-artifact_groups = ["media"]
+artifact_groups = ["media-libs"]
 
 # ==============================================================================
 # ARTIFACT GROUPS - Logical groupings of related artifacts
@@ -285,13 +290,11 @@ type = "generic"
 artifact_group_deps = ["hip-runtime"]
 source_sets = ["iree-libs"]
 
-[artifact_groups.media]
+[artifact_groups.media-libs]
 description = "Media Libraries"
 type = "generic"
 artifact_group_deps = ["third-party-sysdeps", "base"]
-# TODO: rocm-systems included for projects/hip/VERSION (see CMakeLists.txt)
-# amd-mesa disabled on Windows via source_set
-source_sets = ["amd-mesa", "rocm-systems"]
+source_sets = ["amd-mesa", "media-libs"]
 
 # ==============================================================================
 # ARTIFACTS - The actual build outputs that can be transmitted between stages
@@ -308,10 +311,10 @@ artifact_deps = []  # No dependencies on other artifacts
 feature_group = "CORE"  # Part of core, enabled by default
 
 [artifacts.sysdeps-amd-mesa]
-artifact_group = "media"
+artifact_group = "media-libs"
 type = "target-neutral"
 artifact_deps = ["sysdeps"]
-feature_group = "MEDIA"
+feature_group = "MEDIA_LIBS" # Controlled by THEROCK_ENABLE_MEDIA_LIBS
 disable_platforms = ["windows"]
 
 [artifacts.sysdeps-expat]
@@ -525,6 +528,22 @@ type = "target-neutral"
 artifact_deps = ["core-runtime", "core-hip", "hipdnn"]
 disable_platforms = ["windows"]
 
+# --- Media Integration ---
+
+[artifacts.rocdecode]
+artifact_group = "media-libs"
+type = "target-neutral"
+artifact_deps = ["core-runtime", "core-hip", "base", "sysdeps", "sysdeps-amd-mesa"]
+feature_group = "MEDIA_LIBS" # Controlled by THEROCK_ENABLE_MEDIA_LIBS
+disable_platforms = ["windows"]
+
+[artifacts.rocjpeg]
+artifact_group = "media-libs"
+type = "target-neutral"
+artifact_deps = ["core-runtime", "core-hip", "base", "sysdeps", "sysdeps-amd-mesa"]
+feature_group = "MEDIA_LIBS" # Controlled by THEROCK_ENABLE_MEDIA_LIBS
+disable_platforms = ["windows"]
+
 # --- Communication Libraries (per-arch) ---
 
 [artifacts.rccl]

CMakeLists.txt

@@ -236,6 +236,7 @@ option(THEROCK_ENABLE_MATH_LIBS "Enable building of math libraries" "${THEROCK_E
 option(THEROCK_ENABLE_ML_LIBS "Enable building of ML libraries" "${THEROCK_ENABLE_ALL}")
 option(THEROCK_ENABLE_PROFILER "Enable building the profiler libraries" "${THEROCK_ENABLE_ALL}")
 option(THEROCK_ENABLE_DC_TOOLS "Enable building of data center tools" "${THEROCK_ENABLE_ALL}")
+option(THEROCK_ENABLE_MEDIA_LIBS "Enable building of Media libraries" "${THEROCK_ENABLE_ALL}")
 option(THEROCK_ENABLE_HOST_MATH "Build all bundled host math libraries by default" OFF)
 option(THEROCK_RESET_FEATURES "One-shot flag which forces all feature flags to their default state for this configuration run" OFF)
 # TODO(iree-org/fusilli/issues/57): Enable fusilli build once multi-arch support lands.
@@ -372,6 +373,8 @@ set(THEROCK_BUNDLED_NUMACTL)
 set(THEROCK_BUNDLED_SQLITE3)
 set(THEROCK_BUNDLED_ZLIB)
 set(THEROCK_BUNDLED_ZSTD)
+set(THEROCK_BUNDLED_AMDMESA)
+
 
 if(THEROCK_BUNDLE_SYSDEPS)
   message(STATUS "Building with bundled system dependencies enabled")
@@ -398,6 +401,9 @@ if(THEROCK_BUNDLE_SYSDEPS)
     set(THEROCK_BUNDLED_SQLITE3 therock-sqlite3)
     set(THEROCK_BUNDLED_ZLIB therock-zlib)
     set(THEROCK_BUNDLED_ZSTD therock-zstd)
+    if(THEROCK_ENABLE_SYSDEPS_AMD_MESA)
+      set(THEROCK_BUNDLED_AMDMESA therock-amd-mesa)
+    endif()
   elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
     set(THEROCK_BUNDLED_BZIP2 therock-bzip2)
     set(THEROCK_BUNDLED_SQLITE3 therock-sqlite3)
@@ -440,6 +446,9 @@ add_subdirectory(comm-libs)
 add_subdirectory(math-libs)
 add_subdirectory(ml-libs)
 add_subdirectory(iree-libs)
+if(THEROCK_ENABLE_SYSDEPS_AMD_MESA)
+  add_subdirectory(media-libs)
+endif()
 
 if(THEROCK_BUILD_TESTING)
   add_subdirectory(examples)

build_tools/analyze_build_times.py

@@ -64,6 +64,7 @@
     "dctools",
     "profiler",
     "ml-libs",
+    "media-libs",
 }
 
 # Regex to parse artifact filenames: <project>_<variant>[_suffix].tar.xz

build_tools/build_python_packages.py

@@ -74,6 +74,14 @@ def run(args: argparse.Namespace):
         tarball_compression=args.devel_tarball_compression,
     )
 
+    # populate the media package
+    media = PopulatedDistPackage(params, logical_name="media")
+    media.populate_runtime_files(
+        params.filter_artifacts(
+            filter=functools.partial(media_artifact_filter),
+        )
+    )
+
     if args.build_packages:
         build_packages(args.dest_dir, wheel_compression=args.wheel_compression)
 
@@ -126,6 +134,15 @@ def libraries_artifact_filter(target_family: str, an: ArtifactName) -> bool:
     return libraries
 
 
+def media_artifact_filter(an: ArtifactName) -> bool:
+    media = an.name in [
+        "rocdecode",
+        "rocjpeg",
+        "sysdeps-amd-mesa",
+    ]
+    return media
+
+
 def main(argv: list[str]):
     p = argparse.ArgumentParser()
     p.add_argument(

build_tools/detail/linux_portable_build_in_container.sh

@@ -30,6 +30,8 @@ set -o xtrace
 time cmake -GNinja -S /therock/src -B "$OUTPUT_DIR/build" \
   -DTHEROCK_BUNDLE_SYSDEPS=ON \
   -DTHEROCK_ENABLE_SYSDEPS_AMD_MESA=ON \
+  -DTHEROCK_ENABLE_ROCDECODE=ON \
+  -DTHEROCK_ENABLE_ROCJPEG=ON \
   ${PYTHON_EXECUTABLES_ARG} \
   "$@"
 time cmake --build "$OUTPUT_DIR/build" --target therock-archives therock-dist

build_tools/fetch_sources.py

@@ -518,6 +518,8 @@ def main(argv):
             else [
                 # Linux only projects.
                 "amd-mesa",
+                "rocdecode",
+                "rocjpeg",
             ]
         ),
     )

build_tools/github_actions/build_configure.py

@@ -81,6 +81,8 @@ def build_configure(manylinux=False):
         )
         cmd.append(f"-DTHEROCK_DIST_PYTHON_EXECUTABLES={python_executables}")
         cmd.append("-DTHEROCK_ENABLE_SYSDEPS_AMD_MESA=ON")
+        cmd.append("-DTHEROCK_ENABLE_ROCDECODE=ON")
+        cmd.append("-DTHEROCK_ENABLE_ROCJPEG=ON")
 
     if PLATFORM == "windows":
         # VCToolsInstallDir is required for build. Throwing an error if environment variable doesn't exist

build_tools/github_actions/fetch_test_configurations.py

@@ -267,6 +267,24 @@ def _get_script_path(script_name: str) -> str:
         "platform": ["linux"],
         "total_shards": 1,
     },
+    # rocdecode
+    "rocdecode": {
+        "job_name": "rocdecode",
+        "fetch_artifact_args": "--rocdecode",
+        "timeout_minutes": 60,
+        "test_script": f"python {_get_script_path('test_rocdecode.py')}",
+        "platform": ["linux"],
+        "total_shards": 1,
+    },
+    # rocjpeg
+    "rocjpeg": {
+        "job_name": "rocjpeg",
+        "fetch_artifact_args": "--rocjpeg",
+        "timeout_minutes": 60,
+        "test_script": f"python {_get_script_path('test_rocjpeg.py')}",
+        "platform": ["linux"],
+        "total_shards": 1,
+    },
     # aqlprofile tests
     "aqlprofile": {
         "job_name": "aqlprofile",

build_tools/github_actions/test_executable_scripts/test_rocdecode.py

@@ -0,0 +1,80 @@
+import logging
+import os
+import shlex
+import subprocess
+from pathlib import Path
+import sys
+import platform
+
+logging.basicConfig(level=logging.INFO)
+THEROCK_BIN_DIR_STR = os.getenv("THEROCK_BIN_DIR")
+if THEROCK_BIN_DIR_STR is None:
+    logging.info(
+        "++ Error: env(THEROCK_BIN_DIR) is not set. Please set it before executing tests."
+    )
+    sys.exit(1)
+THEROCK_BIN_DIR = Path(THEROCK_BIN_DIR_STR)
+SCRIPT_DIR = Path(__file__).resolve().parent
+THEROCK_DIR = SCRIPT_DIR.parent.parent.parent
+THEROCK_TEST_DIR = Path(THEROCK_DIR) / "build"
+
+ROCDECODE_TEST_PATH = str(
+    Path(THEROCK_BIN_DIR).resolve().parent / "share" / "rocdecode" / "test"
+)
+if not os.path.isdir(ROCDECODE_TEST_PATH):
+    logging.info(f"++ Error: rocdecode tests not found in {ROCDECODE_TEST_PATH}")
+    sys.exit(1)
+else:
+    logging.info(f"++ INFO: rocdecode tests found in {ROCDECODE_TEST_PATH}")
+env = os.environ.copy()
+
+
+# set env variables required for tests
+def setup_env(env):
+    ROCM_PATH = Path(THEROCK_BIN_DIR).resolve().parent
+    env["ROCM_PATH"] = str(ROCM_PATH)
+    logging.info(f"++ rocdecode setting ROCM_PATH={ROCM_PATH}")
+    if platform.system() == "Linux":
+        HIP_LIB_PATH = Path(THEROCK_BIN_DIR).resolve().parent / "lib"
+        logging.info(f"++ rocdecode setting LD_LIBRARY_PATH={HIP_LIB_PATH}")
+        if "LD_LIBRARY_PATH" in env:
+            env["LD_LIBRARY_PATH"] = f"{HIP_LIB_PATH}:{env['LD_LIBRARY_PATH']}"
+        else:
+            env["LD_LIBRARY_PATH"] = str(HIP_LIB_PATH)
+    else:
+        logging.info(f"++ rocdecode tests only supported on Linux")
+        exit()
+
+
+def execute_tests(env):
+    ROCDECODE_TEST_DIR = Path(THEROCK_TEST_DIR) / "rocdecode-test"
+
+    cmd = [
+        "mkdir",
+        "-p",
+        "rocdecode-test",
+    ]
+    logging.info(f"++ Exec [{THEROCK_TEST_DIR}]$ {shlex.join(cmd)}")
+    subprocess.run(cmd, cwd=THEROCK_TEST_DIR, check=True, env=env)
+
+    cmd = [
+        "cmake",
+        "-GNinja",
+        ROCDECODE_TEST_PATH,
+    ]
+
+    logging.info(f"++ Exec [{ROCDECODE_TEST_DIR}]$ {shlex.join(cmd)}")
+    subprocess.run(cmd, cwd=ROCDECODE_TEST_DIR, check=True, env=env)
+
+    cmd = [
+        "ctest",
+        "-VV",
+        "--output-on-failure",
+    ]
+    logging.info(f"++ Exec [{ROCDECODE_TEST_DIR}]$ {shlex.join(cmd)}")
+    subprocess.run(cmd, cwd=ROCDECODE_TEST_DIR, check=True, env=env)
+
+
+if __name__ == "__main__":
+    setup_env(env)
+    execute_tests(env)

build_tools/github_actions/test_executable_scripts/test_rocjpeg.py

@@ -0,0 +1,78 @@
+import logging
+import os
+import shlex
+import subprocess
+from pathlib import Path
+import sys
+import platform
+
+logging.basicConfig(level=logging.INFO)
+THEROCK_BIN_DIR_STR = os.getenv("THEROCK_BIN_DIR")
+if THEROCK_BIN_DIR_STR is None:
+    logging.info(
+        "++ Error: env(THEROCK_BIN_DIR) is not set. Please set it before executing tests."
+    )
+    sys.exit(1)
+THEROCK_BIN_DIR = Path(THEROCK_BIN_DIR_STR)
+SCRIPT_DIR = Path(__file__).resolve().parent
+THEROCK_DIR = SCRIPT_DIR.parent.parent.parent
+THEROCK_TEST_DIR = Path(THEROCK_DIR) / "build"
+
+ROCJPEG_TEST_PATH = str(
+    Path(THEROCK_BIN_DIR).resolve().parent / "share" / "rocjpeg" / "test"
+)
+if not os.path.isdir(ROCJPEG_TEST_PATH):
+    logging.info(f"++ Error: rocjpeg tests not found in {ROCJPEG_TEST_PATH}")
+    sys.exit(1)
+else:
+    logging.info(f"++ INFO: rocjpeg tests found in {ROCJPEG_TEST_PATH}")
+env = os.environ.copy()
+
+
+# set env variables required for tests
+def setup_env(env):
+    ROCM_PATH = Path(THEROCK_BIN_DIR).resolve().parent
+    env["ROCM_PATH"] = str(ROCM_PATH)
+    logging.info(f"++ rocjpeg setting ROCM_PATH={ROCM_PATH}")
+    if platform.system() == "Linux":
+        HIP_LIB_PATH = Path(THEROCK_BIN_DIR).resolve().parent / "lib"
+        logging.info(f"++ rocjpeg setting LD_LIBRARY_PATH={HIP_LIB_PATH}")
+        if "LD_LIBRARY_PATH" in env:
+            env["LD_LIBRARY_PATH"] = f"{HIP_LIB_PATH}:{env['LD_LIBRARY_PATH']}"
+        else:
+            env["LD_LIBRARY_PATH"] = str(HIP_LIB_PATH)
+    else:
+        logging.info(f"++ rocjpeg tests only supported on Linux")
+        exit()
+
+
+def execute_tests(env):
+    ROCJPEG_TEST_DIR = Path(THEROCK_TEST_DIR) / "rocjpeg-test"
+    cmd = [
+        "mkdir",
+        "-p",
+        "rocjpeg-test",
+    ]
+    logging.info(f"++ Exec [{THEROCK_TEST_DIR}]$ {shlex.join(cmd)}")
+    subprocess.run(cmd, cwd=THEROCK_TEST_DIR, check=True, env=env)
+
+    cmd = [
+        "cmake",
+        "-GNinja",
+        ROCJPEG_TEST_PATH,
+    ]
+    logging.info(f"++ Exec [{ROCJPEG_TEST_DIR}]$ {shlex.join(cmd)}")
+    subprocess.run(cmd, cwd=ROCJPEG_TEST_DIR, check=True, env=env)
+
+    cmd = [
+        "ctest",
+        "-VV",
+        "--output-on-failure",
+    ]
+    logging.info(f"++ Exec [{ROCJPEG_TEST_DIR}]$ {shlex.join(cmd)}")
+    subprocess.run(cmd, cwd=ROCJPEG_TEST_DIR, check=True, env=env)
+
+
+if __name__ == "__main__":
+    setup_env(env)
+    execute_tests(env)