Support Intel SDE Tool, improved individual script tests (#333)

* Support subfolder in extract-file

* Enable test for intel sde

* Dont restrict mlc test to console

* Improved github action for individual script tests

Author: arjunsuresh

.github/scripts/list_modified_files.py

@@ -0,0 +1,36 @@
+import yaml
+import sys
+import json
+
+
+def get_num_runs(filepath):
+    with open(filepath, 'r') as file:
+        content = yaml.safe_load(file)
+        tests = content.get('tests', {})
+        if tests:
+            num_tests = len(tests.get('run_inputs', []))
+        else:
+            num_tests = 0
+        uid = content['uid']
+        return uid, num_tests
+
+
+def process_files(files):
+    filenames = files.split()
+    return [
+        {
+            "file": file,
+            "uid": uid,
+            "num_tests": num_tests
+        }
+        for file in filenames
+        if file.endswith('meta.yaml') and (uid := get_num_runs(file))
+        for _, num_tests in [uid]
+    ]
+
+
+if __name__ == "__main__":
+    changed_files = sys.stdin.read().strip()
+    processed_files = process_files(changed_files)
+    json_processed_files = json.dumps(processed_files)
+    print(f"::set-output name=modified_files::{json_processed_files}")

.github/workflows/build_wheel_off.yml

@@ -8,7 +8,6 @@ on:
       - VERSION
 
 jobs:
-
   build_wheels:
     if: github.repository_owner == 'mlcommons'
     name: Build wheel

.github/workflows/build_wheels.yml

@@ -6,7 +6,6 @@ on:
   workflow_dispatch: {}
 
 jobs:
-
   build_wheels:
     if: github.repository_owner == 'mlcommons'
     name: Build wheel

.github/workflows/run-tests-on-modified-meta.yml

@@ -0,0 +1,57 @@
+# This workflow will run configured tests for any updated MLC script
+name: MLC test script on modified meta
+
+on:
+  pull_request:
+    branches: [ "main", "dev" ]
+    paths:
+      - 'script/**meta.yaml'
+
+jobs:
+  get-modified-files:
+    runs-on: ubuntu-latest
+    outputs:
+      modified_files: ${{ steps.modified-files.outputs.modified_files }}
+
+    steps:
+    - name: 'Checkout'
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 2
+   
+    - name: Setup Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.x'
+ 
+    - name: Get changed files
+      id: modified-files
+      run: |
+        git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }}
+        git fetch upstream
+        changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only)
+        echo "$changed_files" | python3 .github/scripts/list_modified_files.py
+
+    
+
+  process_modified_files:
+    runs-on: ubuntu-latest
+    needs: get_modified_files
+    strategy:
+      fail_fast: false
+      matrix:
+        file_info: ${{ fromJSON(needs.get_modified_files.outputs.modified_files) }}
+        num_run: ${{ range(1, fromJSON(matrix.file_info.num_tests) + 1) }}
+    
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Process meta.yaml file
+        run: |
+          echo "Processing ${{ matrix.file_info.file }} with run number ${{ matrix.num_run }}"
+
+          pip install mlcflow
+          mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
+          mlc test script ${{ matrix.file_info.uid}} --test_input_index=${{ matrix.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet
+

automation/script/module.py

@@ -2929,7 +2929,7 @@ def test(self, i):
 
             alias = meta.get('alias', '')
             uid = meta.get('uid', '')
-            if console:
+            if console or True:  # Todo restrict to console only?
                 logger.info(path)
                 test_config = meta.get('tests', '')
                 if test_config:

script/download-and-extract/meta.yaml

@@ -19,6 +19,7 @@ input_mapping:
 new_env_keys:
 - MLC_DOWNLOAD_DOWNLOADED_PATH*
 - MLC_EXTRACT_EXTRACTED_PATH
+- MLC_EXTRACT_EXTRACTED_SUBDIR_PATH
 - <<<MLC_DOWNLOAD_FINAL_ENV_NAME>>>
 - <<<MLC_EXTRACT_FINAL_ENV_NAME>>>
 - <<<MLC_DAE_FINAL_ENV_NAME>>>

script/download-file/customize.py

@@ -138,7 +138,7 @@ def preprocess(i):
                     shell=True,
                     env=subprocess_env)
             if (env.get('MLC_DOWNLOAD_CHECKSUM_FILE', '') != '' or env.get(
-                    'MLC_DOWNLOAD_CHECKSUM', '') != '') and os.path.isfile(env['MLC_DOWNLOAD_FILENAME']):
+                    'MLC_DOWNLOAD_CHECKSUM', '') != '') and os.path.exists(env['MLC_DOWNLOAD_FILENAME']):
                 # print(checksum_result) #for debugging
                 if "checksum did not match" in checksum_result.stderr.lower():
                     computed_checksum = subprocess.run(

script/extract-file/customize.py

@@ -176,11 +176,11 @@ def postprocess(i):
 
     extract_to_folder = env.get('MLC_EXTRACT_TO_FOLDER', '')
     extract_path = env.get('MLC_EXTRACT_PATH', '')
-
+    folderpath = None
     extracted_file = env.get('MLC_EXTRACT_EXTRACTED_FILENAME', '')
 
     # Preparing filepath
-    #   Can be either full extracted filename (such as model) or folder
+    # Can be either the full extracted filename (such as model file) or folder
 
     if extracted_file != '':
         filename = os.path.basename(extracted_file)
@@ -191,8 +191,8 @@ def postprocess(i):
 
         filepath = os.path.join(folderpath, filename)
     else:
-
         filepath = os.getcwd()  # Extracted to the root cache folder
+        folderpath = os.getcwd()
 
     if not os.path.exists(filepath):
         return {
@@ -215,6 +215,18 @@ def postprocess(i):
         if archive_filepath != '' and os.path.isfile(archive_filepath):
             os.remove(archive_filepath)
 
+    # Check if only a single folder is created and if so, export the folder
+    # name
+    if folderpath:
+        sub_items = os.listdir(folderpath)
+        sub_folders = [
+            item for item in sub_items if os.path.isdir(
+                os.path.join(
+                    folderpath, item))]
+        if len(sub_folders) == 1:
+            env['MLC_EXTRACT_EXTRACTED_SUBDIR_PATH'] = os.path.join(
+                folderpath, sub_folders[0])
+
     # Since may change directory, check if need to clean some temporal files
     automation.clean_some_tmp_files({'env': env})
 

script/extract-file/meta.yaml

@@ -23,6 +23,7 @@ input_mapping:
   to: MLC_EXTRACT_PATH
 new_env_keys:
 - MLC_EXTRACT_EXTRACTED_PATH
+- MLC_EXTRACT_EXTRACTED_SUBDIR_PATH
 - <<<MLC_EXTRACT_FINAL_ENV_NAME>>>
 - MLC_GET_DEPENDENT_CACHED_PATH
 new_state_keys: []

script/get-aocc/customize.py

@@ -65,7 +65,6 @@ def preprocess(i):
                                            'run_script_input': i['run_script_input'],
                                            'recursion_spaces': i['recursion_spaces']})
         if r['return'] > 0:
-
             return r
 
     return {'return': 0}