review changes

Author: mgumowsk

.github/components-path-filters.yml

@@ -61,6 +61,11 @@ platform/services/platform_cleaner:
   - Makefile.shared-python
   - Makefile.shared
   - .github/workflows/component.yml
+platform/services/weights_uploader:
+  - platform/services/weights_uploader/**
+  - Makefile.shared-python
+  - Makefile.shared
+  - .github/workflows/component.yml
 
 # IAI Services
 interactive_ai/services/auto_train:

interactive_ai/workflows/geti_domain/common/jobs_common/k8s_helpers/trainer_pod_definition.py

@@ -229,7 +229,7 @@ def create_flyte_container_task(  # noqa: PLR0913
                     V1EnvVar(name="TASK_ID", value=container_name),
                     V1EnvVar(name="SESSION_ORGANIZATION_ID", value=str(session.organization_id)),
                     V1EnvVar(name="SESSION_WORKSPACE_ID", value=str(session.workspace_id)),
-                    V1EnvVar(name="WEIGHTS_URL", value="https://storage.geti.infra-host.com"),
+                    V1EnvVar(name="WEIGHTS_URL", value="https://download.geti-dev.infra-host.com/weights"),
                     V1EnvVar(
                         name="KAFKA_TOPIC_PREFIX",
                         value_from=V1EnvVarSource(

platform/services/weights_uploader/app/pretrained_models/pretrained_models.py

@@ -5,7 +5,6 @@
 import logging
 import os
 import shutil
-import ssl
 import urllib.error
 import urllib.request
 import zipfile
@@ -29,15 +28,6 @@ def sha256sum(filepath: str):  # noqa: ANN201, D103
     return sha256.hexdigest()
 
 
-def get_file(url: str, target_path: str, auto_unzip=True):  # noqa: ANN001, ANN201, D103
-    target_url = url
-    try:
-        download_file(target_url, target_path, auto_unzip)
-    except Exception:
-        logger.info("cannot find the file from local mirror. try to get it from the original url")
-        download_file(url, target_path, auto_unzip)
-
-
 def download_file(url: str, target_path: str, auto_unzip=True):  # noqa: ANN001, ANN201, D103
     logger.info(f"Downloading file: {url}")
     url_original_filename = os.path.basename(url)
@@ -47,13 +37,8 @@ def download_file(url: str, target_path: str, auto_unzip=True):  # noqa: ANN001,
     target_dir_path = os.path.dirname(target_path)
     download_temp_target_path = os.path.join(target_dir_path, url_original_filename)
 
-    # Disable SSL check
-    ctx = ssl.create_default_context()
-    ctx.check_hostname = False
-    ctx.verify_mode = ssl.CERT_NONE
-
     with (
-        urllib.request.urlopen(url, context=ctx) as response,  # noqa: S310
+        urllib.request.urlopen(url) as response,  # noqa: S310
         open(download_temp_target_path, "wb") as out_file,
     ):
         shutil.copyfileobj(response, out_file)
@@ -89,7 +74,7 @@ def retry_call(call: Callable, retries: int = RETRIES, **kwargs):  # noqa: ANN20
             call(**kwargs)
             break
         except Exception:
-            logging.exception(f"Failed try {i + 1}/{retries}")
+            logger.exception(f"Failed try {i + 1}/{retries}")
     else:
         raise MaxTriesExhausted
 
@@ -106,19 +91,19 @@ def download_pretrained_model(model_spec: dict, target_dir: str, weights_url: st
 
     if os.path.exists(target_download_path):
         if sha_sum is None:
-            logging.warning(f"Model already existed: {target_download_path} but sha_sum is not specified")
-            logging.warning(f"consider to add sha_sum to the model spec: {sha256sum(target_download_path)}")
+            logger.warning(f"Model already existed: {target_download_path} but sha_sum is not specified")
+            logger.warning(f"consider to add sha_sum to the model spec: {sha256sum(target_download_path)}")
         elif sha256sum(target_download_path) == sha_sum:
             logger.info(f"Model already downloaded: {target_download_path}")
             return
         else:
-            logging.warning(f"Model already downloaded but SHA mismatch: {target_download_path}")
-            logging.warning("Redownloading...")
+            logger.warning(f"Model already downloaded but SHA mismatch: {target_download_path}")
+            logger.warning("Redownloading...")
             os.remove(target_download_path)
 
     try:
         retry_call(
-            get_file,
+            download_file,
             url=model_external_url,
             target_path=target_download_path,
             auto_unzip=auto_unzip,

platform/services/weights_uploader/app/pretrained_models/pretrained_models_v2.json

@@ -92,6 +92,7 @@
   {
     "url": "https://github.com/lyuwenyu/storage/releases/download/v0.1/ResNet18_vd_pretrained_from_paddle.pth",
     "target": "/home/non-root/.cache/torch/hub/checkpoints/ResNet18_vd_pretrained_from_paddle.pth",
+    "sha_sum": "911a745b62e173c8f4b9af513c2ea295428cf23f1bbfe9048381500f140fd720",
     "template_ids": ["Object_Detection_RTDetr_18"]
   },
   {
@@ -192,16 +193,19 @@
   {
     "url": "https://github.com/lyuwenyu/storage/releases/download/v0.1/ResNet101_vd_ssld_pretrained_from_paddle.pth",
     "target": "/home/non-root/.cache/torch/hub/checkpoints/ResNet101_vd_ssld_pretrained_from_paddle.pth",
+    "sha_sum": "3683aca21d46f3a78de3171ee8f0497a9b1b4cd139a254e81b3d5bf579a0ff78",
     "template_ids": ["Object_Detection_RTDetr_101"]
   },
   {
     "url": "https://github.com/lyuwenyu/storage/releases/download/v0.1/ResNet50_vd_ssld_v2_pretrained_from_paddle.pth",
     "target": "/home/non-root/.cache/torch/hub/checkpoints/ResNet50_vd_ssld_v2_pretrained_from_paddle.pth",
+    "sha_sum": "641dc59835d5228e6013ac45acb633592ec825144fb3d78055bc07699f404353",
     "template_ids": ["Object_Detection_RTDetr_50"]
   },
   {
     "url": "https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/cspnext-tiny_udp-aic-coco_210e-256x192-cbed682d_20230130.pth",
-    "target": "/home/non-root/.cache/torch/hub/checkpoints/cspnext-tiny_udp-aic-coco_210e-256x192-cbed682d_20230130.pth"
+    "target": "/home/non-root/.cache/torch/hub/checkpoints/cspnext-tiny_udp-aic-coco_210e-256x192-cbed682d_20230130.pth",
+    "sha_sum": "69c5e24fd2c8340efd3ffbce4ca37d7e04e905414887fc624736d665dfec4b58",
   },
   {
     "url": "https://github.com/osmr/imgclsmob/releases/download/v0.0.364/efficientnet_b0-0752-0e386130.pth.zip",
@@ -247,7 +251,8 @@
   },
   {
     "url": "https://download.pytorch.org/models/maskrcnn_resnet50_fpn_v2_coco-73cbd019.pth",
-    "target": "/home/non-root/.cache/torch/hub/checkpoints/maskrcnn_resnet50_fpn_v2_coco-73cbd019.pth"
+    "target": "/home/non-root/.cache/torch/hub/checkpoints/maskrcnn_resnet50_fpn_v2_coco-73cbd019.pth",
+    "sha_sum": "73cbd0190fcbe3ba339921fbce2c3a0b6bb9126c9a133c85e43a2a8e060a109e",
   },
   {
     "url": "https://download.pytorch.org/models/resnet50-0676ba61.pth",
@@ -257,7 +262,8 @@
   },
   {
     "url": "https://github.com/ChaoningZhang/MobileSAM/raw/master/weights/mobile_sam.pt",
-    "target": "/home/non-root/.cache/torch/hub/checkpoints/mobile_sam.pt"
+    "target": "/home/non-root/.cache/torch/hub/checkpoints/mobile_sam.pt",
+    "sha_sum": "6dbb90523a35330fedd7f1d3dfc66f995213d81b29a5ca8108dbcdd4e37d6c2f",
   },
   {
     "url": "https://storage.openvinotoolkit.org/repositories/openvino_training_extensions/models/object_detection/v2/mobilenet_v2-2s_ssd-992x736.pth",
@@ -286,6 +292,7 @@
   {
     "url": "https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet-ins_tiny_8xb32-300e_coco/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth",
     "target": "/home/non-root/.cache/torch/hub/checkpoints/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth",
+    "sha_sum": "ec670f7ee9e20bd7931e15f15b7016f7fe531baaab81f2e6153382d046111885",
     "template_ids": ["Custom_Instance_Segmentation_RTMDet_tiny"]
   },
   {
@@ -323,6 +330,7 @@
   {
     "url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_coco.pth",
     "target": "/home/non-root/.cache/torch/hub/checkpoints/dfine_x_coco.pth",
+    "sha_sum": "9a82252f2d0426b2c24e99905c36c33a4ea850f247c247bda76fb5f76daea123",
     "template_ids": ["Object_Detection_DFine_X"]
   },
   {