Merge branch 'dev' into ghaction-big-sized-calib

Author: anandhu-eng

automation/script/module.py

@@ -82,7 +82,8 @@ def __init__(self, action_object, automation_file):
                                              'skip_system_deps',
                                              'git_ssh',
                                              'gh_token',
-                                             'hf_token']
+                                             'hf_token',
+                                             'verify_ssl']
 
     ############################################################
 

automation/utils.py

@@ -178,10 +178,10 @@ def download_file(i):
 
     text = i.get('text', 'Downloaded: ')
 
-    if 'MLC_UTILS_DOWNLOAD_VERIFY_SSL' in os.environ:
-        verify = os.environ['MLC_UTILS_DOWNLOAD_VERIFY_SSL'] == 'yes'
+    if 'MLC_VERIFY_SSL' in os.environ:
+        verify = is_true(os.environ['MLC_VERIFY_SSL'])
     else:
-        verify = i.get('verify', True)
+        verify = is_true(i.get('verify_ssl', True))
 
     try:
         with requests.get(url, stream=True, allow_redirects=True, verify=verify) as download:

script/download-and-extract/meta.yaml

@@ -15,7 +15,6 @@ input_mapping:
   store: MLC_DOWNLOAD_PATH
   to: MLC_EXTRACT_PATH
   url: MLC_DAE_URL
-  verify: MLC_VERIFY_SSL
 new_env_keys:
 - MLC_DOWNLOAD_DOWNLOADED_PATH*
 - MLC_EXTRACT_EXTRACTED_PATH

script/download-file/customize.py

@@ -84,8 +84,8 @@ def preprocess(i):
 
         extra_download_options = env.get('MLC_DOWNLOAD_EXTRA_OPTIONS', '')
 
-        verify_ssl = env.get('MLC_VERIFY_SSL', "True")
-        if is_false(verify_ssl) or os_info['platform'] == 'windows':
+        verify_ssl = is_true(env.get('MLC_VERIFY_SSL', "True"))
+        if not verify_ssl or os_info['platform'] == 'windows':
             verify_ssl = False
         else:
             verify_ssl = True
@@ -229,17 +229,17 @@ def preprocess(i):
             rclone_copy_using = env.get('MLC_RCLONE_COPY_USING', 'sync')
             if rclone_copy_using == "sync":
                 pre_clean = False
+            if not verify_ssl:
+                extra_download_options += " --no-check-certificate "
             if env["MLC_HOST_OS_TYPE"] == "windows":
                 # have to modify the variable from url to temp_url if it is
                 # going to be used anywhere after this point
                 url = url.replace("%", "%%")
                 temp_download_file = env['MLC_DOWNLOAD_FILENAME'].replace(
                     "%", "%%")
-                env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P --error-on-no-transfer"
+                env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P --error-on-no-transfer {extra_download_options}"
             else:
                 env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['MLC_DOWNLOAD_FILENAME'])}{q} -P --error-on-no-transfer {extra_download_options}"
-            if not verify_ssl:
-                env['MLC_DOWNLOAD_CMD'] += f" --no-check-certificate"
 
         filename = env['MLC_DOWNLOAD_FILENAME']
         env['MLC_DOWNLOAD_DOWNLOADED_FILENAME'] = filename

script/download-file/meta.yaml

@@ -24,8 +24,6 @@ input_mapping:
   output_file: MLC_DOWNLOAD_FILENAME
   store: MLC_DOWNLOAD_PATH
   url: MLC_DOWNLOAD_URL
-  verify: MLC_VERIFY_SSL
-  verify_ssl: MLC_VERIFY_SSL
 new_env_keys:
 - MLC_DOWNLOAD_DOWNLOADED_PATH
 - <<<MLC_DOWNLOAD_FINAL_ENV_NAME>>>

script/get-ml-model-gptj/meta.yaml

@@ -95,7 +95,6 @@ variations:
   pytorch,fp32:
     env:
       MLC_DOWNLOAD_CHECKSUM_NOT_USED: e677e28aaf03da84584bb3073b7ee315
-      MLC_DOWNLOAD_EXTRA_OPTIONS: ' --output-document checkpoint.zip'
       MLC_PACKAGE_URL: https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx/download
       MLC_RCLONE_CONFIG_NAME: mlc-inference
       MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/gpt-j
@@ -105,6 +104,8 @@ variations:
     add_deps_recursive:
       dae:
         tags: _extract
+    env:
+      MLC_DOWNLOAD_EXTRA_OPTIONS: ' --output-document checkpoint.zip'
   pytorch,int4,intel: {}
   pytorch,int8,intel: {}
   pytorch,intel: