Merge branch 'dev' into dev

Author: amd-arsuresh

.github/workflows/test-mlc-script-features.yml

@@ -156,6 +156,81 @@ jobs:
 
         print("\nAll process_mounts tests passed!")
         PYTEST
+    - name: Test nested Docker opt-in mount handling
+      shell: bash
+      run: |
+        python - <<'PY'
+        import importlib.util
+        import sys
+        from types import SimpleNamespace
+        from unittest.mock import patch
+
+        sys.path.insert(0, 'automation')
+
+        spec = importlib.util.spec_from_file_location(
+            'run_docker_customize',
+            'script/run-docker-container/customize.py'
+        )
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+
+        logger_messages = []
+        logger = SimpleNamespace(
+            info=lambda *args, **kwargs: logger_messages.append(('info', args)),
+            error=lambda *args, **kwargs: logger_messages.append(('error', args))
+        )
+        DOCKER_ENV_MARKER = '/.dockerenv'
+        DOCKER_SOCKET = '/var/run/docker.sock'
+        DOCKER_BIN = '/usr/bin/docker'
+
+        def run_case(enable_nested, docker_exit_code=0):
+            captured = {'cmd': ''}
+
+            def fake_system(cmd):
+                captured['cmd'] = cmd
+                return docker_exit_code
+
+            def fake_exists(path):
+                if path in [DOCKER_ENV_MARKER, DOCKER_SOCKET, DOCKER_BIN]:
+                    return True
+                return False
+
+            env = {
+                'MLC_CONTAINER_TOOL': 'docker',
+                'MLC_DOCKER_RUN_SCRIPT_TAGS': 'run,docker,container',
+                'MLC_DOCKER_RUN_CMD': 'echo nested-docker-ci-check',
+                'MLC_DOCKER_ENABLE_NESTED': 'yes' if enable_nested else 'no'
+            }
+            i = {
+                'os_info': {'platform': 'linux'},
+                'env': env,
+                'automation': SimpleNamespace(logger=logger)
+            }
+
+            with patch.object(mod.os.path, 'exists', side_effect=fake_exists), \
+                    patch.object(mod.os, 'system', side_effect=fake_system), \
+                    patch.object(mod.shutil, 'which', return_value=DOCKER_BIN):
+                result = mod.postprocess(i)
+
+            return result, captured['cmd']
+
+        enabled_result, enabled_cmd = run_case(True, docker_exit_code=0)
+        disabled_result, disabled_cmd = run_case(False, docker_exit_code=0)
+        failed_result, _ = run_case(True, docker_exit_code=1)
+
+        assert enabled_result['return'] == 0, enabled_result
+        assert disabled_result['return'] == 0, disabled_result
+        error_msg = failed_result.get('error', '').lower()
+        assert failed_result['return'] == 1, failed_result
+        assert 'docker' in error_msg, failed_result
+        assert 'failed' in error_msg, failed_result
+
+        assert f'{DOCKER_SOCKET}:{DOCKER_SOCKET}' in enabled_cmd, enabled_cmd
+        assert f'{DOCKER_BIN}:{DOCKER_BIN}' in enabled_cmd, enabled_cmd
+        assert f'{DOCKER_SOCKET}:{DOCKER_SOCKET}' not in disabled_cmd, disabled_cmd
+        assert f'{DOCKER_BIN}:{DOCKER_BIN}' not in disabled_cmd, disabled_cmd
+        print('Nested Docker opt-in mount handling test passed')
+        PY
 
   test_experiment:
     runs-on: ${{ matrix.os }}

.github/workflows/test-mlperf-inference-resnet50.yml

@@ -1,5 +1,8 @@
 name: MLPerf inference ResNet50
 
+permissions:
+  contents: read
+
 on:
   pull_request_target:
     branches: [ "main", "dev" ]
@@ -107,4 +110,45 @@ jobs:
         git config --global credential.https://gist.github.com.helper ""
         git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
         mlcr push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
+ 
+  mlc-run-rhel-docker:
+    runs-on: ubuntu-latest
+    env:
+      MLC_INDEX: "on"
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.12"
+    - name: Install mlcflow
+      run: |
+        pip install mlcflow
+        pip install tabulate
+    - name: Pull MLOps repo
+      shell: bash
+      env:
+        REPO: ${{ github.event.pull_request.head.repo.html_url }}
+        BRANCH: ${{ github.event.pull_request.head.ref }}
+      run: |
+        mlc pull repo "$REPO" --branch="$BRANCH"
+    - name: Test MLPerf Inference ResNet50 (RHEL Docker)
+      run: |
+        mlcd run-mlperf,inference,_submission,_short,_r6.0-dev \
+          --submitter="MLCommons" \
+          --pull_changes=yes \
+          --pull_inference_changes=yes \
+          --hw_name="gh_rhel9_docker x86" \
+          --model=resnet50 \
+          --implementation=python \
+          --backend=onnxruntime \
+          --device=cpu \
+          --scenario=Offline \
+          --test_query_count=500 \
+          --target_qps=1 \
+          --docker_os=rhel \
+          --docker_os_version=9 \
+          -v --quiet
    

.gitignore

@@ -5,8 +5,10 @@ MANIFEST
 __pycache__
 develop-eggs/
 dist/
+*.egg-info/
 eggs/
 .eggs/
+*.egg-info/
 lib/
 lib64/
 sdist/

automation/script/README.md

@@ -788,6 +788,7 @@ mlcr "app,mlperf,inference" --docker_run --docker_image_repo=myrepo --docker_reb
 3. **Mount Handling**: Host directories are automatically mounted into the container based on environment variable paths.
 4. **Environment Passing**: Host env variables (proxies, tokens, etc.) are passed into the container.
 5. **Execution**: The MLC run command is re-executed inside the container with `--docker_run_deps` to handle Docker-specific dependency installation.
+6. **Nested Docker Support**: Set `MLC_DOCKER_ENABLE_NESTED=yes` to mount Docker socket (`/var/run/docker.sock`) and Docker CLI binary into launched containers (when available) when MLC is running inside Docker.
 
 ### Docker Meta Configuration
 

automation/script/docker.py

@@ -25,10 +25,10 @@ def dockerfile(self_module, input_params):
     # Step 2: Process Dockerfile-related configurations
     env = input_params.get('env', {})
 
-    host_only_env_keys = [ "HOME", "USER" ]
+    host_only_env_keys = ["HOME", "USER"]
     for key in host_only_env_keys:
         if key in env:
-            del(env[key])
+            del (env[key])
 
     state_data = input_params.get('state', {})
     constant_vars = input_params.get('const', {})
@@ -141,8 +141,9 @@ def dockerfile(self_module, input_params):
         comments = []
 
     # Push Docker image if specified
-    if str(input_params.get('docker_push_image')
-           ).lower() in ['true', 'yes', '1']:
+    docker_push_image = input_params.get(
+        'docker_push_image', input_params.get('docker_upload', ''))
+    if str(docker_push_image).lower() in ['true', 'yes', '1']:
         env['MLC_DOCKER_PUSH_IMAGE'] = 'yes'
 
     dockerfile_env = docker_inputs.get('env')
@@ -229,10 +230,10 @@ def docker_run(self_module, i):
     show_time = i.get('show_time', False)
     logger = self_module.logger
     env = i.get('env', {})
-    host_only_env_keys = [ "HOME", "USER" ]
+    host_only_env_keys = ["HOME", "USER"]
     for key in host_only_env_keys:
         if key in env:
-            del(env[key])
+            del (env[key])
 
     self_module.env = env
 

automation/script/docker_utils.py

@@ -142,6 +142,7 @@ def prepare_docker_inputs(input_params, docker_settings,
         "mlc_repos", "skip_mlc_sys_upgrade", "extra_sys_deps", "image_name",
         "gh_token", "fake_run_deps", "run_final_cmds", "real_run", "copy_files", "path", "user", "env", "build_env"
     ]
+    keys += ["cache", "split_mlc_run_cmd"]
 
     if run_stage:
         keys += [

script/README.md

@@ -923,6 +923,9 @@ This directory contains automation scripts for MLPerf benchmarks, AI/ML workflow
 - **[get-mlperf-training-src](get-mlperf-training-src/)**
   - get-mlperf-training-src
   - Tags: `get`, `src`, `source`, `training`, `training-src`, `training-source`, `mlperf`, `mlcommons`
+- **[get-mlperf-training-results-summary](get-mlperf-training-results-summary/)**
+  - get-mlperf-training-results-summary
+  - Tags: `get`, `mlperf`, `training`, `results`, `summary`, `summary-results`, `results-summary`
 - **[prepare-training-data-bert](prepare-training-data-bert/)**
   - prepare-training-data-bert
   - Tags: `prepare`, `mlperf`, `training`, `data`, `input`, `bert`

script/app-mlperf-inference-mlcommons-cpp/customize.py

@@ -77,7 +77,8 @@ def preprocess(i):
         env['+ CXXFLAGS'].append("-std=c++14")
 
     # add preprocessor flag like "#define MLC_MODEL_RESNET50"
-    env['+ CXXFLAGS'].append('-DMLC_MODEL_' + env['MLC_MODEL'].upper().replace('-', '_').replace('.', '_'))
+    env['+ CXXFLAGS'].append('-DMLC_MODEL_' +
+                             env['MLC_MODEL'].upper().replace('-', '_').replace('.', '_'))
     # add preprocessor flag like "#define MLC_MLPERF_BACKEND_ONNXRUNTIME"
     env['+ CXXFLAGS'].append('-DMLC_MLPERF_BACKEND_' +
                              env['MLC_MLPERF_BACKEND'].upper())
@@ -90,7 +91,13 @@ def preprocess(i):
         import torch as _torch
         torch_path = os.path.dirname(_torch.__file__)
         torch_inc = os.path.join(torch_path, 'include')
-        torch_inc_csrc = os.path.join(torch_path, 'include', 'torch', 'csrc', 'api', 'include')
+        torch_inc_csrc = os.path.join(
+            torch_path,
+            'include',
+            'torch',
+            'csrc',
+            'api',
+            'include')
         torch_lib = os.path.join(torch_path, 'lib')
         env['+CPLUS_INCLUDE_PATH'].append(torch_inc)
         env['+CPLUS_INCLUDE_PATH'].append(torch_inc_csrc)
@@ -126,7 +133,6 @@ def preprocess(i):
     env['MLC_LINKER_LANG'] = 'CXX'
     env['MLC_RUN_DIR'] = os.getcwd()
 
-
     # For PyTorch backend, convert .pth weights to TorchScript .pt if needed
     if env.get('MLC_MLPERF_BACKEND', '') == 'pytorch':
         model_path = env.get('MLC_ML_MODEL_FILE_WITH_PATH', '')
@@ -135,9 +141,14 @@ def preprocess(i):
             if not os.path.exists(torchscript_path):
                 import torch
                 import torchvision.models as models
-                logger.info(f"Converting {model_path} to TorchScript at {torchscript_path}")
+                logger.info(
+                    f"Converting {model_path} to TorchScript at {torchscript_path}")
                 model = models.resnet50()
-                model.load_state_dict(torch.load(model_path, map_location='cpu', weights_only=False))
+                model.load_state_dict(
+                    torch.load(
+                        model_path,
+                        map_location='cpu',
+                        weights_only=False))
                 model.eval()
                 traced = torch.jit.trace(model, torch.randn(1, 3, 224, 224))
                 traced.save(torchscript_path)