[Housekeeping] Fixing relative imports for better usability and clarity

hot_fair_utilities/__init__.py

@@ -1,7 +1,7 @@
-from .georeferencing import georeference
-from .inference import evaluate, predict
-from .postprocessing import polygonize, vectorize
-from .preprocessing import preprocess, yolo_v8_v1
+from hot_fair_utilities.georeferencing import georeference
+from hot_fair_utilities.inference import evaluate, predict
+from hot_fair_utilities.postprocessing import polygonize, vectorize
+from hot_fair_utilities.preprocessing import preprocess, yolo_v8_v1
 
-# from .training import ramp, yolo_v8_v1
-from .utils import bbox2tiles, tms2img
+# from hot_fair_utilities.training import ramp, yolo_v8_v1
+from hot_fair_utilities.utils import bbox2tiles, tms2img

hot_fair_utilities/georeferencing.py

@@ -8,7 +8,7 @@
 from osgeo import gdal
 from tqdm import tqdm
 
-from .utils import get_bounding_box
+from hot_fair_utilities.utils import get_bounding_box
 
 
 def georeference(input_path: str, output_path: str, is_mask=False,epsg=3857) -> None:

hot_fair_utilities/inference/__init__.py

@@ -1,2 +1,2 @@
-from .predict import predict
-from .evaluate import evaluate
+from hot_fair_utilities.inference.predict import predict
+from hot_fair_utilities.inference.evaluate import evaluate

hot_fair_utilities/inference/evaluate.py

@@ -8,7 +8,7 @@
 
 try:
     # Third party imports
-    from ramp.utils.eval_utils import get_iou_accuracy_metrics
+    from ramp_fair.utils.eval_utils import get_iou_accuracy_metrics
 except ImportError:
     print("Ramp eval metrics are not available, Possibly ramp is not installed")
 

hot_fair_utilities/preprocessing/__init__.py

@@ -1,2 +1,2 @@
-from .preprocess import preprocess
-from .yolo_v8_v1 import yolo_format
+from hot_fair_utilities.preprocessing import preprocess
+from hot_fair_utilities.preprocessing.yolo_v8_v1 import yolo_format

hot_fair_utilities/preprocessing/clip_labels.py

@@ -10,7 +10,7 @@
 from shapely.geometry import box
 from tqdm import tqdm
 
-from ..utils import get_bounding_box
+from hot_fair_utilities.utils import get_bounding_box
 
 
 def clip_labels(

hot_fair_utilities/preprocessing/fix_labels.py

@@ -29,9 +29,13 @@ def fix_labels(input_path: str, output_path: str) -> None:
         input_path: Path to the GeoJSON file where the input data are stored.
         output_path: Path to the GeoJSON file where the output data will go.
     """
-    gdf = geopandas.read_file(os.path.relpath(input_path))
+    print(f"Fixing labels in {input_path}")
+    # gdf = geopandas.read_file(os.path.relpath(input_path))
+    gdf = geopandas.read_file(input_path)
     # print(gdf)
     if gdf.empty:
         raise ValueError("Error: gdf is empty, No Labels found : Check your labels")
     gdf["geometry"] = gdf.apply(remove_self_intersection, axis=1)
     gdf.to_file(output_path, driver="GeoJSON")
+
+    return

hot_fair_utilities/preprocessing/multimasks_from_polygons.py

@@ -6,12 +6,12 @@
 # Third party imports
 import geopandas as gpd
 import rasterio as rio
-from ramp.data_mgmt.chip_label_pairs import (
-    construct_mask_filepath,
-    get_tq_chip_label_pairs,
-)
-from ramp.utils.img_utils import to_channels_first
-from ramp.utils.multimask_utils import df_to_px_mask, multimask_to_sparse_multimask
+
+from ramp_fair.data_mgmt.chip_label_pairs import construct_mask_filepath
+from ramp_fair.data_mgmt.chip_label_pairs import get_tq_chip_label_pairs
+
+from ramp_fair.utils.img_utils import to_channels_first
+from ramp_fair.utils.multimask_utils import df_to_px_mask, multimask_to_sparse_multimask
 from solaris.utils.core import _check_rasterio_im_load
 from solaris.utils.geo import get_crs
 from solaris.vector.mask import crs_is_metric
@@ -96,13 +96,23 @@ def multimasks_from_polygons(
 
         reference_im = _check_rasterio_im_load(chip_path)
 
-        if get_crs(gdf) != get_crs(reference_im):
-            # BUGFIX: if crs's don't match, reproject the geodataframe
-            gdf = gdf.to_crs(get_crs(reference_im))
+        #check co-ordinate systems
+        gdf_crs = get_crs(gdf)
+        ref_crs = get_crs(reference_im)
+
+        # Only attempt reprojection if both CRS values are valid and not equal.
+        if gdf_crs is None or ref_crs is None:
+            raise ValueError("Invalid CRS encountered in input data.")
+        if gdf_crs != ref_crs:
+            try:
+                gdf = gdf.to_crs(ref_crs)
+                print("Reprojected GeoDataFrame to reference CRS.")
+            except Exception as e:
+                print("Error reprojecting GeoDataFrame:", e)
+                raise
 
         if crs_is_metric(gdf):
             meters = True
-
             boundary_width = min(reference_im.res) * input_boundary_width
             contact_spacing = min(reference_im.res) * input_contact_spacing
 

hot_fair_utilities/preprocessing/preprocess.py

@@ -1,10 +1,10 @@
 # Standard library imports
 import os
 
-from ..georeferencing import georeference
-from .clip_labels import clip_labels
-from .fix_labels import fix_labels
-from .reproject_labels import reproject_labels_to_epsg3857
+from hot_fair_utilities.georeferencing import georeference
+from hot_fair_utilities.preprocessing.clip_labels import clip_labels
+from hot_fair_utilities.preprocessing.fix_labels import fix_labels
+from hot_fair_utilities.preprocessing.reproject_labels import reproject_labels_to_epsg3857
 
 
 def preprocess(
@@ -110,7 +110,7 @@ def preprocess(
         os.remove(f"{output_path}/labels_epsg3857.geojson")
 
     if multimasks:
-        from .multimasks_from_polygons import multimasks_from_polygons
+        from hot_fair_utilities.preprocessing.multimasks_from_polygons import multimasks_from_polygons
 
         assert os.path.isdir(
             f"{output_path}/chips"

hot_fair_utilities/training/ramp/__init__.py

@@ -1 +1 @@
-from .train import train, run_feedback
+from hot_fair_utilities.training.train import train, run_feedback

hot_fair_utilities/training/ramp/run_training.py

@@ -36,10 +36,12 @@ def __init__(self, message):
 
 # Third party imports
 import segmentation_models as sm
+
 from ramp.data_mgmt.data_generator import (
     test_batches_from_gtiff_dirs,
     training_batches_from_gtiff_dirs,
 )
+
 from ramp.training import (
     callback_constructors,
     loss_constructors,
@@ -52,14 +54,15 @@ def __init__(self, message):
 from ramp.training.augmentation_constructors import get_augmentation_fn
 from ramp.utils.misc_ramp_utils import get_num_files
 
-from .config import RAMP_CONFIG
+from hot_fair_utilities.training.ramp.config import RAMP_CONFIG
 
 # Segmentation Models: using `keras` framework.
 sm.set_framework("tf.keras")
 
 
 # this variable must be defined. It is the parent of the 'ramp-code' directory.
-working_ramp_home = os.environ["RAMP_HOME"]
+# working_ramp_home = os.environ["RAMP_HOME"]
+repo_home = os.system("git rev-parse --show-toplevel")
 
 
 def apply_feedback(

hot_fair_utilities/training/ramp/train.py

@@ -1,9 +1,9 @@
 # Standard library imports
 import os
 
-from .cleanup import extract_highest_accuracy_model
-from .prepare_data import split_training_2_validation
-from .run_training import apply_feedback, manage_fine_tuning_config, run_main_train_code
+from hot_fair_utilities.training.ramp.cleanup import extract_highest_accuracy_model
+from hot_fair_utilities.training.ramp.prepare_data import split_training_2_validation
+from hot_fair_utilities.training.ramp.run_training import apply_feedback, manage_fine_tuning_config, run_main_train_code
 
 
 def train(
@@ -83,7 +83,7 @@ def run_feedback(
     assert os.path.exists(input_path), "Input Feedback Path Doesn't Exist"
     assert os.path.exists(feedback_base_model), "Feedback base Model Doesn't Exist"
     os.environ.update(os.environ)
-    os.environ["RAMP_HOME"] = model_home
+    # os.environ["RAMP_HOME"] = model_home
     print("Starting to prepare data for training")
     split_training_2_validation(input_path, output_path, multimasks)
     print("Data is ready for training")

hot_fair_utilities/utils.py

@@ -14,7 +14,6 @@
 # Third party imports
 # Third-party imports
 import geopandas
-import matplotlib.pyplot as plt
 import pandas as pd
 import requests
 import ultralytics
@@ -257,7 +256,6 @@ def fetch_osm_data(payload: json, API_URL="https://api-prod.raw-data.hotosm.org/
 
 
 # Third party imports
-import matplotlib.pyplot as plt
 import pandas as pd
 
 

requirements.txt

@@ -17,6 +17,7 @@ black
 isort
 build
 twine
+protobuf==3.19.6
 # torch==1.12.1
 # torchvision==0.13.1
 # ultralytics==8.1.6

setup.py

@@ -1,5 +1,10 @@
-# Third party imports
-from setuptools import find_packages, setup
+from setuptools import setup, find_packages
+import os
+
+if os.path.exists('./ramp-code'):
+    pass
+else:
+    print('[WARNING] - ramp-code folder does not exist, some functionalities may not work correctly.')
 
 setup(
     package_dir={"": "."},

test_ramp.py

@@ -9,51 +9,81 @@
 print(
     f"\nUsing tensorflow version {tf. __version__} with no of gpu : {len(tf.config.experimental.list_physical_devices('GPU'))}\n"
 )
-print(os.getcwd())
-os.environ.update(os.environ)
+# print(os.getcwd())
+# os.environ.update(os.environ)
+
 # Add a new environment variable to the operating system
-os.environ["RAMP_HOME"] = os.getcwd()
+# os.environ["RAMP_HOME"] = os.getcwd()
 # Print the environment variables to verify that the new variable was added
-print(os.environ["RAMP_HOME"])
+# print(os.environ["RAMP_HOME"])
 
 start_time = time.time()
 # Third party imports
 # %%
 # import ramp.utils
 
 # Reader imports
-import hot_fair_utilities
-
-base_path = f"{os.getcwd()}/ramp-data/sample_2"
+# import hot_fair_utilities
+top_level_repo_folder_path = str(os.system("git rev-parse --show-toplevel"))
+sample_ramp_data_folder_path = os.path.join(top_level_repo_folder_path,'ramp-data/sample_2')
 
 # Reader imports
 # %%
 from hot_fair_utilities import preprocess
 
-model_input_image_path = f"{base_path}/input"
-preprocess_output = f"/{base_path}/preprocessed"
-preprocess(
-    input_path=model_input_image_path,
-    output_path=preprocess_output,
-    rasterize=True,
-    rasterize_options=["binary"],
-    georeference_images=True,
-    multimasks=True,
-)
+ramp_sample_input_folder = os.path.join(sample_ramp_data_folder_path,'input')
+# model_input_image_path = f"{base_path}/input"
+ramp_sample_preprocessed_folder = os.path.join(sample_ramp_data_folder_path,'preprocessed')
+# preprocess_output = f"/{base_path}/preprocessed"
+
+def test_preprocess_binary():
+    preprocess(
+        input_path=ramp_sample_input_folder,
+        output_path=ramp_sample_preprocessed_folder,
+        rasterize=True,
+        rasterize_options=["binary"],
+        georeference_images=True,
+        multimasks=False
+    )
+    print("Preprocessing with binary masks completed successfully.")
+
+    return
+
+test_preprocess_binary()
+
+def test_preprocess_multimask():
+    preprocess(
+        input_path=ramp_sample_input_folder,
+        output_path=ramp_sample_preprocessed_folder,
+        rasterize=True,
+        rasterize_options=["binary"],
+        georeference_images=True,
+        multimasks=True
+    )
+
+    print("Preprocessing with multi-masks completed successfully.")
+
+    return
+
+test_preprocess_multimask()
 
 # Reader imports
 # %%
 from hot_fair_utilities.training.ramp import train
 
 # %%
-train_output = f"{base_path}/train"
+# train_output = f"{base_path}/train"
+ramp_sample_ready_for_training_folder = os.path.join(sample_ramp_data_folder_path,'ready_for_trainining')
+
+
 final_accuracy, final_model_path = train(
-    input_path=preprocess_output,
-    output_path=train_output,
+    input_path=ramp_sample_preprocessed_folder,
+    output_path=ramp_sample_ready_for_training_folder,
     epoch_size=2,
     batch_size=2,
     model="ramp",
-    model_home=os.environ["RAMP_HOME"],
+    # model_home=os.environ["RAMP_HOME"],
+    model_home=top_level_repo_folder_path
 )
 
 # %%
@@ -63,10 +93,13 @@
 # %%
 from hot_fair_utilities import predict
 
-prediction_output = f"{base_path}/prediction/output"
+# prediction_output = f"{base_path}/prediction/output"
+prediction_input = os.path.join(sample_ramp_data_folder_path,'prediction/input')
+prediiction_output = os.path.join(sample_ramp_data_folder_path,'prediction/output')
+
 predict(
     checkpoint_path=final_model_path,
-    input_path=f"{base_path}/prediction/input",
+    input_path=prediction_input,
     prediction_path=prediction_output,
 )