ignore some files during upload

Author: Dpbm

export/huggingface.py

@@ -8,6 +8,7 @@
 
 from utils.datatypes import FilePath
 from export.helpers import get_latest_model
+from utils.constants import MODEL_FILE_PREFIX, CHECKPOINT_FILE_PREFIX
 
 def upload_dataset(folder:FilePath):
     """
@@ -21,7 +22,13 @@ def upload_dataset(folder:FilePath):
         folder_path=folder,
         repo_id=dataset_name,
         repo_type="dataset",
-        ignore_patterns=["dataset/"]
+        ignore_patterns=[
+            "dataset/", 
+            "*.json", 
+            f"{MODEL_FILE_PREFIX}*", 
+            f"{CHECKPOINT_FILE_PREFIX}*", 
+            "*.png",
+        ]
     )
 
 def upload_model(folder:str):

export/kaggle.py

@@ -8,6 +8,7 @@
 import kagglehub as kh
 
 from utils.datatypes import FilePath
+from utils.constants import MODEL_FILE_PREFIX, CHECKPOINT_FILE_PREFIX
 
 def upload_dataset(folder:FilePath):
     """
@@ -20,7 +21,13 @@ def upload_dataset(folder:FilePath):
         dataset_name, 
         folder, 
         version_notes=version,
-        ignore_patterns=["dataset/"]
+        ignore_patterns=[
+            "dataset/", 
+            "*.json", 
+            f"{MODEL_FILE_PREFIX}*", 
+            f"{CHECKPOINT_FILE_PREFIX}*", 
+            "*.png",
+        ]
     )
 
 def upload_model(folder:str):
@@ -35,7 +42,16 @@ def upload_model(folder:str):
         handle=model_name,
         local_model_dir=folder,
         version_notes=version,
-        ignore_patterns=["dataset/","ghz*","*.zip","*.h5","*.csv"]
+        ignore_patterns=[
+            "dataset/",
+            "ghz*",
+            "*.pth"
+            "*.zip",
+            "*.h5",
+            "*.csv",
+            "*.png",
+            "*.json"
+        ]
     )
 
 if __name__ == "__main__":

train.py

@@ -19,6 +19,7 @@
 from utils.constants import (
     DEBUG,
     MODEL_FILE_PREFIX,
+    CHECKPOINT_FILE_PREFIX,
     dataset_file,
     images_h5_file,
     ghz_file,
@@ -265,7 +266,7 @@ def save(
         scheduler: StateDict,
     ):
         """Save checkpoint data"""
-        path = os.path.join(folder, "checkpoint_%s.pth" % (time.ctime()))
+        path = os.path.join(folder, "%s%s.pth" % (CHECKPOINT_FILE_PREFIX, time.ctime()))
         print("%sSaving checkpoint at: %s...%s" % (Colors.MAGENTABG, path, Colors.ENDC))
         checkpoint = {
             "epoch": epoch,

utils/constants.py

@@ -26,6 +26,7 @@
 DEFAULT_CHECKPOINT = None
 
 MODEL_FILE_PREFIX = "model_"
+CHECKPOINT_FILE_PREFIX = "checkpoint_"
 
 # ruff: noqa: E731
 dataset_path = lambda target_folder: os.path.join(target_folder, "dataset")