Skip to content

Commit c2961ee

Browse files
mgumowskmgumowsk
committed
review changes
1 parent e98d273 commit c2961ee

File tree

3 files changed

+19
-26
lines changed

3 files changed

+19
-26
lines changed

‎interactive_ai/workflows/geti_domain/common/jobs_common/k8s_helpers/trainer_pod_definition.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ def create_flyte_container_task( # noqa: PLR0913
229229
V1EnvVar(name="TASK_ID", value=container_name),
230230
V1EnvVar(name="SESSION_ORGANIZATION_ID", value=str(session.organization_id)),
231231
V1EnvVar(name="SESSION_WORKSPACE_ID", value=str(session.workspace_id)),
232-
V1EnvVar(name="WEIGHTS_URL", value="https://storage.geti.infra-host.com"),
232+
V1EnvVar(name="WEIGHTS_URL", value="https://download.geti-dev.infra-host.com/weights"),
233233
V1EnvVar(
234234
name="KAFKA_TOPIC_PREFIX",
235235
value_from=V1EnvVarSource(

‎platform/services/weights_uploader/app/pretrained_models/pretrained_models.py

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import logging
66
import os
77
import shutil
8-
import ssl
98
import urllib.error
109
import urllib.request
1110
import zipfile
@@ -29,15 +28,6 @@ def sha256sum(filepath: str): # noqa: ANN201, D103
2928
return sha256.hexdigest()
3029

3130

32-
def get_file(url: str, target_path: str, auto_unzip=True): # noqa: ANN001, ANN201, D103
33-
target_url = url
34-
try:
35-
download_file(target_url, target_path, auto_unzip)
36-
except Exception:
37-
logger.info("cannot find the file from local mirror. try to get it from the original url")
38-
download_file(url, target_path, auto_unzip)
39-
40-
4131
def download_file(url: str, target_path: str, auto_unzip=True): # noqa: ANN001, ANN201, D103
4232
logger.info(f"Downloading file: {url}")
4333
url_original_filename = os.path.basename(url)
@@ -47,13 +37,8 @@ def download_file(url: str, target_path: str, auto_unzip=True): # noqa: ANN001,
4737
target_dir_path = os.path.dirname(target_path)
4838
download_temp_target_path = os.path.join(target_dir_path, url_original_filename)
4939

50-
# Disable SSL check
51-
ctx = ssl.create_default_context()
52-
ctx.check_hostname = False
53-
ctx.verify_mode = ssl.CERT_NONE
54-
5540
with (
56-
urllib.request.urlopen(url, context=ctx) as response, # noqa: S310
41+
urllib.request.urlopen(url) as response, # noqa: S310
5742
open(download_temp_target_path, "wb") as out_file,
5843
):
5944
shutil.copyfileobj(response, out_file)
@@ -89,7 +74,7 @@ def retry_call(call: Callable, retries: int = RETRIES, **kwargs): # noqa: ANN20
8974
call(**kwargs)
9075
break
9176
except Exception:
92-
logging.exception(f"Failed try {i + 1}/{retries}")
77+
logger.exception(f"Failed try {i + 1}/{retries}")
9378
else:
9479
raise MaxTriesExhausted
9580

@@ -106,19 +91,19 @@ def download_pretrained_model(model_spec: dict, target_dir: str, weights_url: st
10691

10792
if os.path.exists(target_download_path):
10893
if sha_sum is None:
109-
logging.warning(f"Model already existed: {target_download_path} but sha_sum is not specified")
110-
logging.warning(f"consider to add sha_sum to the model spec: {sha256sum(target_download_path)}")
94+
logger.warning(f"Model already existed: {target_download_path} but sha_sum is not specified")
95+
logger.warning(f"consider to add sha_sum to the model spec: {sha256sum(target_download_path)}")
11196
elif sha256sum(target_download_path) == sha_sum:
11297
logger.info(f"Model already downloaded: {target_download_path}")
11398
return
11499
else:
115-
logging.warning(f"Model already downloaded but SHA mismatch: {target_download_path}")
116-
logging.warning("Redownloading...")
100+
logger.warning(f"Model already downloaded but SHA mismatch: {target_download_path}")
101+
logger.warning("Redownloading...")
117102
os.remove(target_download_path)
118103

119104
try:
120105
retry_call(
121-
get_file,
106+
download_file,
122107
url=model_external_url,
123108
target_path=target_download_path,
124109
auto_unzip=auto_unzip,

‎platform/services/weights_uploader/app/pretrained_models/pretrained_models_v2.json

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@
9292
{
9393
"url": "https://github.com/lyuwenyu/storage/releases/download/v0.1/ResNet18_vd_pretrained_from_paddle.pth",
9494
"target": "/home/non-root/.cache/torch/hub/checkpoints/ResNet18_vd_pretrained_from_paddle.pth",
95+
"sha_sum": "911a745b62e173c8f4b9af513c2ea295428cf23f1bbfe9048381500f140fd720",
9596
"template_ids": ["Object_Detection_RTDetr_18"]
9697
},
9798
{
@@ -192,16 +193,19 @@
192193
{
193194
"url": "https://github.com/lyuwenyu/storage/releases/download/v0.1/ResNet101_vd_ssld_pretrained_from_paddle.pth",
194195
"target": "/home/non-root/.cache/torch/hub/checkpoints/ResNet101_vd_ssld_pretrained_from_paddle.pth",
196+
"sha_sum": "3683aca21d46f3a78de3171ee8f0497a9b1b4cd139a254e81b3d5bf579a0ff78",
195197
"template_ids": ["Object_Detection_RTDetr_101"]
196198
},
197199
{
198200
"url": "https://github.com/lyuwenyu/storage/releases/download/v0.1/ResNet50_vd_ssld_v2_pretrained_from_paddle.pth",
199201
"target": "/home/non-root/.cache/torch/hub/checkpoints/ResNet50_vd_ssld_v2_pretrained_from_paddle.pth",
202+
"sha_sum": "641dc59835d5228e6013ac45acb633592ec825144fb3d78055bc07699f404353",
200203
"template_ids": ["Object_Detection_RTDetr_50"]
201204
},
202205
{
203206
"url": "https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/cspnext-tiny_udp-aic-coco_210e-256x192-cbed682d_20230130.pth",
204-
"target": "/home/non-root/.cache/torch/hub/checkpoints/cspnext-tiny_udp-aic-coco_210e-256x192-cbed682d_20230130.pth"
207+
"target": "/home/non-root/.cache/torch/hub/checkpoints/cspnext-tiny_udp-aic-coco_210e-256x192-cbed682d_20230130.pth",
208+
"sha_sum": "69c5e24fd2c8340efd3ffbce4ca37d7e04e905414887fc624736d665dfec4b58",
205209
},
206210
{
207211
"url": "https://github.com/osmr/imgclsmob/releases/download/v0.0.364/efficientnet_b0-0752-0e386130.pth.zip",
@@ -247,7 +251,8 @@
247251
},
248252
{
249253
"url": "https://download.pytorch.org/models/maskrcnn_resnet50_fpn_v2_coco-73cbd019.pth",
250-
"target": "/home/non-root/.cache/torch/hub/checkpoints/maskrcnn_resnet50_fpn_v2_coco-73cbd019.pth"
254+
"target": "/home/non-root/.cache/torch/hub/checkpoints/maskrcnn_resnet50_fpn_v2_coco-73cbd019.pth",
255+
"sha_sum": "73cbd0190fcbe3ba339921fbce2c3a0b6bb9126c9a133c85e43a2a8e060a109e",
251256
},
252257
{
253258
"url": "https://download.pytorch.org/models/resnet50-0676ba61.pth",
@@ -257,7 +262,8 @@
257262
},
258263
{
259264
"url": "https://github.com/ChaoningZhang/MobileSAM/raw/master/weights/mobile_sam.pt",
260-
"target": "/home/non-root/.cache/torch/hub/checkpoints/mobile_sam.pt"
265+
"target": "/home/non-root/.cache/torch/hub/checkpoints/mobile_sam.pt",
266+
"sha_sum": "6dbb90523a35330fedd7f1d3dfc66f995213d81b29a5ca8108dbcdd4e37d6c2f",
261267
},
262268
{
263269
"url": "https://storage.openvinotoolkit.org/repositories/openvino_training_extensions/models/object_detection/v2/mobilenet_v2-2s_ssd-992x736.pth",
@@ -286,6 +292,7 @@
286292
{
287293
"url": "https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet-ins_tiny_8xb32-300e_coco/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth",
288294
"target": "/home/non-root/.cache/torch/hub/checkpoints/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth",
295+
"sha_sum": "ec670f7ee9e20bd7931e15f15b7016f7fe531baaab81f2e6153382d046111885",
289296
"template_ids": ["Custom_Instance_Segmentation_RTMDet_tiny"]
290297
},
291298
{
@@ -323,6 +330,7 @@
323330
{
324331
"url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_x_coco.pth",
325332
"target": "/home/non-root/.cache/torch/hub/checkpoints/dfine_x_coco.pth",
333+
"sha_sum": "9a82252f2d0426b2c24e99905c36c33a4ea850f247c247bda76fb5f76daea123",
326334
"template_ids": ["Object_Detection_DFine_X"]
327335
},
328336
{

0 commit comments

Comments
 (0)