Merge branch 'mlcommons:dev' into dev

Author: amd-arsuresh

.github/workflows/build_wheel_off.yml

@@ -7,6 +7,7 @@ on:
     paths:
       - VERSION
 
+
 jobs:
   build_wheels:
     if: github.repository_owner == 'mlcommons'

script/get-dataset-mmlu/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2026-2027 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/get-dataset-mmlu/customize.py

@@ -0,0 +1,35 @@
+from mlc import utils
+from utils import is_true
+import os
+
+
+def preprocess(i):
+
+    os_info = i['os_info']
+
+    env = i['env']
+
+    if env.get('MLC_DATASET_DOWNLOAD_SRC', '') != "mlcommons":
+        print("Using MLCommons Inference source from '" +
+              env['MLC_MLPERF_INFERENCE_SOURCE'] + "'")
+        downloader_path = os.path.join(
+            env['MLC_MLPERF_INFERENCE_SOURCE'], 'automotive', 'llm', 'download_data.py')
+        env['MLC_DATASET_MMLU_PATH'] = os.path.join(
+            env.get('MLC_DATASET_MMLU_OUT_PATH', os.getcwd()), "mmlu.json")
+        run_cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} {downloader_path} --output {env['MLC_DATASET_MMLU_PATH']}"
+        if env.get('MLC_DATASET_MMLU_COUNT', '') != '':
+            run_cmd += f" --count {int(env['MLC_DATASET_MMLU_COUNT'])})"
+
+    env['MLC_RUN_CMD'] = run_cmd
+
+    return {'return': 0}
+
+
+def postprocess(i):
+
+    env = i['env']
+
+    print(
+        f"Path to the MMLU dataset: {env['MLC_DATASET_MMLU_PATH']}")
+
+    return {'return': 0}

script/get-dataset-mmlu/meta.yaml

@@ -0,0 +1,38 @@
+alias: get-dataset-mmlu
+automation_alias: script
+automation_uid: 5b4e0237da074764
+cache: true
+deps:
+- names:
+  - python
+  tags: get,python
+  skip_if_env:
+    MLC_DATASET_DOWNLOAD_SRC: mlcommons
+- names:
+  - automotive-src
+  tags: mlperf,automotive,source
+  skip_if_env:
+    MLC_DATASET_DOWNLOAD_SRC: mlcommons # skip if download happens from mlc infrastructure
+- tags: get,generic-python-lib,_package.tqdm
+  skip_if_env:
+    MLC_DATASET_DOWNLOAD_SRC: mlcommons
+- tags: get,generic-python-lib,_package.datasets
+  skip_if_env:
+    MLC_DATASET_DOWNLOAD_SRC: mlcommons
+input_mapping:
+  out_path: MLC_DATASET_MMLU_OUT_PATH
+new_env_keys:
+- MLC_DATASET_MMLU_PATH
+tags:
+- get-dataset-mmlu
+uid: 137c805565214b95
+variations:
+  # calibration:
+  #   env:
+  #     MLC_IGBH_CALIBRATION_FLAG: --calibration
+  no-of-samples.#:
+    env:
+      MLC_DATASET_MMLU_COUNT: '#'
+  mlc:
+    env:
+      MLC_DATASET_DOWNLOAD_SRC: mlcommons

script/get-dataset-mmlu/run.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH}
+
+#To export any variable
+#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out
+
+#${MLC_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency
+
+
+
+function exit_if_error() {
+  test $? -eq 0 || exit $?
+}
+
+function run() {
+  echo "Running: "
+  echo "$1"
+  echo ""
+  eval "$1"
+  exit_if_error
+}
+
+run "$MLC_RUN_CMD"

script/get-git-repo/customize.py

@@ -1,16 +1,14 @@
 from mlc import utils
 import os
 import shutil
+from datetime import datetime
+import re
 
 
 def preprocess(i):
 
     os_info = i['os_info']
 
-#    if os_info['platform'] == 'windows':
-# return {'return':1, 'error': 'Windows is not supported in this script
-# yet'}
-
     env = i['env']
     meta = i['meta']
 
@@ -83,10 +81,11 @@ def postprocess(i):
 
     env['MLC_GET_DEPENDENT_CACHED_PATH'] = git_checkout_path
 
-    if os.path.exists("tmp-mlc-git-hash.out"):
-        with open("tmp-mlc-git-hash.out", "r") as f:
-            git_hash = f.readline().strip()
-            env['MLC_GIT_REPO_CURRENT_HASH'] = git_hash
+    version_string = generate_src_version_string(env)
+    env['MLC_GIT_REPO_VERSION_STRING'] = version_string
+
+    if (env.get('MLC_GIT_REPO_VERSION_ENV_NAME', '') != ''):
+        env[env['MLC_GIT_REPO_VERSION_ENV_NAME']] = version_string
 
     return {'return': 0}
 
@@ -109,3 +108,50 @@ def update_env(env, key, env_key, var):
         env[key.format(env_key)] = var
 
     return
+
+
+def generate_src_version_string(env):
+    """
+    Generates a descriptive version string including checkout state,
+    date, short hash, and modifiers (PRs, cherry-picks, patches).
+    """
+
+    # 1. Base Components
+    checkout = env.get("MLC_GIT_CHECKOUT", "unknown")
+    sha = env.get("MLC_GIT_SHA", "00000000")
+
+    # Generate the date string (e.g., 20260224)
+    # This uses the current build/checkout date.
+    date_str = datetime.now().strftime("%Y%m%d")
+
+    # Sanitize checkout string (e.g., "feature/login" -> "feature-login")
+    safe_checkout = re.sub(r'[^a-zA-Z0-9]', '-', checkout).strip('-').lower()
+    short_sha = sha[:8]
+
+    # Start building the string parts: Branch + Date + Hash
+    version_parts = [f"{safe_checkout}-{date_str}-g{short_sha}"]
+
+    # 2. Add Pull Request info
+    pr = env.get("MLC_GIT_APPLIED_PR")
+    if pr:
+        # Extract just the numbers from the PR string
+        pr_num = re.search(r'\d+', pr)
+        pr_str = pr_num.group(0) if pr_num else "pr"
+        version_parts.append(f"pr{pr_str}")
+
+    # 3. Add Cherry-pick info
+    cps = env.get("MLC_GIT_APPLIED_CHERRYPICKS")
+    if cps:
+        cp_list = [cp for cp in cps.split(';') if cp.strip()]
+        if cp_list:
+            version_parts.append(f"cp{len(cp_list)}")
+
+    # 4. Add Patch info
+    patches = env.get("MLC_GIT_APPLIED_PATCHES")
+    if patches:
+        patch_list = [p for p in patches.split(';') if p.strip()]
+        if patch_list:
+            version_parts.append(f"p{len(patch_list)}")
+
+    # Assemble the final string
+    return "-".join(version_parts)

script/get-git-repo/meta.yaml

@@ -25,6 +25,7 @@ new_env_keys:
 - MLC_GIT_CHECKOUT_PATH
 - MLC_GIT_REPO_*
 - <<<MLC_GIT_CHECKOUT_PATH_ENV_NAME>>>
+- <<<MLC_GIT_REPO_VERSION_ENV_NAME>>>
 post_deps:
 - dynamic: true
   enable_if_env:

script/get-git-repo/run.bat

@@ -1,70 +1,120 @@
 @echo off
-
-rem echo ******************************************************
-rem echo Cloning MLCommons from %MLC_GIT_URL% with branch %MLC_GIT_CHECKOUT% %MLC_GIT_DEPTH% %MLC_GIT_RECURSE_SUBMODULES% ...
-
-rem git clone %MLC_GIT_RECURSE_SUBMODULES% %MLC_GIT_URL% %MLC_GIT_DEPTH% inference
-rem cd inference
-rem git checkout -b "%MLC_GIT_CHECKOUT%"
-rem
-
-rem Next line allows ERRORLEVEL inside if statements!
 setlocal enabledelayedexpansion
 
-set CUR_DIR=%cd%
-set SCRIPT_DIR=%MLC_TMP_CURRENT_SCRIPT_PATH%
+set "CUR_DIR=%CD%"
+echo %CUR_DIR%
+set "SCRIPT_DIR=%MLC_TMP_CURRENT_SCRIPT_PATH%"
+set "ENV_OUT_FILE=%CUR_DIR%\tmp-run-env.out"
 
-set folder=%MLC_GIT_CHECKOUT_FOLDER%
+set "folder=%MLC_GIT_CHECKOUT_FOLDER%"
 
 if not exist "%MLC_TMP_GIT_PATH%" (
+    if exist "%folder%" (
+        echo rmdir /s /q "%folder%"
+        rmdir /s /q "%folder%"
+    )
+    echo ******************************************************
+    echo Current directory: %CUR_DIR%
+    echo.
+    echo Cloning %MLC_GIT_REPO_NAME% from %MLC_GIT_URL%
+    echo.
+    echo %MLC_GIT_CLONE_CMD%
+    echo.
+
+    %MLC_GIT_CLONE_CMD%
+    
+    :: Retry clone once on failure
+    if ERRORLEVEL 1 (
+        if exist "%folder%" rmdir /s /q "%folder%"
+        %MLC_GIT_CLONE_CMD%
+        if ERRORLEVEL 1 exit /b !ERRORLEVEL!
+    )
+
+    :: /d ensures drive letter changes if necessary
+    cd /d "%folder%"
 
-  if exist "%folder%" (
-    rmdir /S /Q "%folder%"  rem Use rmdir instead of deltree
-  )
-  
-  echo ******************************************************
-  echo Current directory: %CUR_DIR%
-  echo.
-  echo Cloning %MLC_GIT_REPO_NAME% from %MLC_GIT_URL%
-  echo.
-  echo "%MLC_GIT_CLONE_CMD%"
-  echo.
-  
-  %MLC_GIT_CLONE_CMD%
-  IF !ERRORLEVEL! NEQ 0 EXIT !ERRORLEVEL!
-  
-  cd "%folder%"
-  
-  if not "%MLC_GIT_SHA%" == "" (
-       echo.
-       echo.
-       git checkout "%MLC_GIT_CHECKOUT%"
-       IF !ERRORLEVEL! NEQ 0 EXIT !ERRORLEVEL!
-  )
+    if not "%MLC_GIT_SHA%"=="" (
+        echo.
+        echo git checkout -b %MLC_GIT_SHA% %MLC_GIT_SHA%
+        git checkout -b %MLC_GIT_SHA% %MLC_GIT_SHA%
+        if ERRORLEVEL 1 exit /b !ERRORLEVEL!
 
+    ) else if not "%MLC_GIT_CHECKOUT_TAG%"=="" (
+        echo.
+        echo git fetch --all --tags
+        git fetch --all --tags
+        echo git checkout tags/%MLC_GIT_CHECKOUT_TAG% -b %MLC_GIT_CHECKOUT_TAG%
+        git checkout tags/%MLC_GIT_CHECKOUT_TAG% -b %MLC_GIT_CHECKOUT_TAG%
+        if ERRORLEVEL 1 exit /b !ERRORLEVEL!
+    )
 ) else (
-  cd "%folder%"
+    cd /d "%folder%"
 )
 
-if not "%MLC_GIT_SUBMODULES%" == "" (
-  for /F %%s in ("%MLC_GIT_SUBMODULES%") do (
+:: ---------------------------------------------------------
+:: Capture Checkout & SHA information
+:: ---------------------------------------------------------
+:: Determine the active branch name
+for /f "delims=" %%i in ('git rev-parse --abbrev-ref HEAD') do set "ACTUAL_CHECKOUT=%%i"
+
+:: Fallback to detached HEAD hash
+if "%ACTUAL_CHECKOUT%"=="HEAD" (
+    for /f "delims=" %%i in ('git rev-parse HEAD') do set "ACTUAL_CHECKOUT=%%i"
+)
+
+for /f "delims=" %%i in ('git rev-parse HEAD') do set "CURRENT_SHA=%%i"
+
+:: Write to out file (No space before >> prevents trailing spaces in value)
+echo MLC_GIT_CHECKOUT=!ACTUAL_CHECKOUT!>>"%ENV_OUT_FILE%"
+echo MLC_GIT_SHA=!CURRENT_SHA!>>"%ENV_OUT_FILE%"
+
+:: ---------------------------------------------------------
+:: Apply PR, Cherry-picks, and Patches
+:: ---------------------------------------------------------
+if not "%MLC_GIT_PR_TO_APPLY%"=="" (
     echo.
-    echo Initializing submodule %%s
-    git submodule update --init %%s
-    IF !ERRORLEVEL! NEQ 0 EXIT !ERRORLEVEL!
-  )
+    echo Fetching from %MLC_GIT_PR_TO_APPLY%
+    git fetch origin %MLC_GIT_PR_TO_APPLY%:tmp-apply
+    
+    :: Log the PR applied
+    echo MLC_GIT_APPLIED_PR=%MLC_GIT_PR_TO_APPLY%>>"%ENV_OUT_FILE%"
 )
 
-if "%MLC_GIT_PATCH%" == "yes" (
-   for %%x in (%MLC_GIT_PATCH_FILEPATHS%) do (
-       echo.
-       echo Applying patch %%x ...
-       git apply %%x
-       IF !ERRORLEVEL! NEQ 0 EXIT !ERRORLEVEL!
-   )
+if not "%MLC_GIT_CHERRYPICKS%"=="" (
+    :: Log the cherry-picks applied
+    echo MLC_GIT_APPLIED_CHERRYPICKS=%MLC_GIT_CHERRYPICKS%>>"%ENV_OUT_FILE%"
+    
+    :: Replace semicolons with quoted spaces to safely loop in Windows
+    set "CPS="%MLC_GIT_CHERRYPICKS:;=" "%""
+    for %%c in (!CPS!) do (
+        echo.
+        echo Applying cherrypick %%~c
+        git cherry-pick -n %%~c
+        if ERRORLEVEL 1 exit /b !ERRORLEVEL!
+    )
 )
 
-cd "%CUR_DIR%"
+if not "%MLC_GIT_SUBMODULES%"=="" (
+    set "SUBS="%MLC_GIT_SUBMODULES:;=" "%""
+    for %%s in (!SUBS!) do (
+        echo.
+        echo Initializing submodule %%~s
+        git submodule update --init --recursive --checkout --force "%%~s"
+        if ERRORLEVEL 1 exit /b !ERRORLEVEL!
+    )
+)
 
-exit /b 0
+if "%MLC_GIT_PATCH%"=="yes" (
+    :: Log the patches applied
+    echo MLC_GIT_APPLIED_PATCHES=%MLC_GIT_PATCH_FILEPATHS%>>"%ENV_OUT_FILE%"
+    
+    set "PATCHES="%MLC_GIT_PATCH_FILEPATHS:;=" "%""
+    for %%p in (!PATCHES!) do (
+        echo.
+        echo Applying patch %%~p
+        git apply "%%~p"
+        if ERRORLEVEL 1 exit /b !ERRORLEVEL!
+    )
+)
 
+cd /d "%CUR_DIR%"