diff --git a/hot_fair_utilities/__init__.py b/hot_fair_utilities/__init__.py index f4bb24ac..a3ac957b 100644 --- a/hot_fair_utilities/__init__.py +++ b/hot_fair_utilities/__init__.py @@ -1,7 +1,7 @@ -from .georeferencing import georeference -from .inference import evaluate, predict -from .postprocessing import polygonize, vectorize -from .preprocessing import preprocess, yolo_v8_v1 +from hot_fair_utilities.georeferencing import georeference +from hot_fair_utilities.inference import evaluate, predict +from hot_fair_utilities.postprocessing import polygonize, vectorize +from hot_fair_utilities.preprocessing import preprocess, yolo_v8_v1 -# from .training import ramp, yolo_v8_v1 -from .utils import bbox2tiles, tms2img +# from hot_fair_utilities.training import ramp, yolo_v8_v1 +from hot_fair_utilities.utils import bbox2tiles, tms2img diff --git a/hot_fair_utilities/georeferencing.py b/hot_fair_utilities/georeferencing.py index b6a1e556..62fcfea2 100644 --- a/hot_fair_utilities/georeferencing.py +++ b/hot_fair_utilities/georeferencing.py @@ -8,7 +8,7 @@ from osgeo import gdal from tqdm import tqdm -from .utils import get_bounding_box +from hot_fair_utilities.utils import get_bounding_box def georeference(input_path: str, output_path: str, is_mask=False,epsg=3857) -> None: diff --git a/hot_fair_utilities/inference/__init__.py b/hot_fair_utilities/inference/__init__.py index ea947c3b..20a4cd0e 100644 --- a/hot_fair_utilities/inference/__init__.py +++ b/hot_fair_utilities/inference/__init__.py @@ -1,2 +1,2 @@ -from .predict import predict -from .evaluate import evaluate +from hot_fair_utilities.inference.predict import predict +from hot_fair_utilities.inference.evaluate import evaluate diff --git a/hot_fair_utilities/inference/evaluate.py b/hot_fair_utilities/inference/evaluate.py index f1188ffd..e7b72280 100644 --- a/hot_fair_utilities/inference/evaluate.py +++ b/hot_fair_utilities/inference/evaluate.py @@ -8,7 +8,7 @@ try: # Third party imports - from ramp.utils.eval_utils import get_iou_accuracy_metrics + from ramp_fair.utils.eval_utils import get_iou_accuracy_metrics except ImportError: print("Ramp eval metrics are not available, Possibly ramp is not installed") diff --git a/hot_fair_utilities/preprocessing/__init__.py b/hot_fair_utilities/preprocessing/__init__.py index 6a146dcb..dd08ddef 100644 --- a/hot_fair_utilities/preprocessing/__init__.py +++ b/hot_fair_utilities/preprocessing/__init__.py @@ -1,2 +1,2 @@ -from .preprocess import preprocess -from .yolo_v8_v1 import yolo_format +from hot_fair_utilities.preprocessing import preprocess +from hot_fair_utilities.preprocessing.yolo_v8_v1 import yolo_format diff --git a/hot_fair_utilities/preprocessing/clip_labels.py b/hot_fair_utilities/preprocessing/clip_labels.py index 8af073ab..8d5cc038 100644 --- a/hot_fair_utilities/preprocessing/clip_labels.py +++ b/hot_fair_utilities/preprocessing/clip_labels.py @@ -10,7 +10,7 @@ from shapely.geometry import box from tqdm import tqdm -from ..utils import get_bounding_box +from hot_fair_utilities.utils import get_bounding_box def clip_labels( diff --git a/hot_fair_utilities/preprocessing/fix_labels.py b/hot_fair_utilities/preprocessing/fix_labels.py index eb3c25d4..58409c2d 100644 --- a/hot_fair_utilities/preprocessing/fix_labels.py +++ b/hot_fair_utilities/preprocessing/fix_labels.py @@ -29,9 +29,13 @@ def fix_labels(input_path: str, output_path: str) -> None: input_path: Path to the GeoJSON file where the input data are stored. output_path: Path to the GeoJSON file where the output data will go. """ - gdf = geopandas.read_file(os.path.relpath(input_path)) + print(f"Fixing labels in {input_path}") + # gdf = geopandas.read_file(os.path.relpath(input_path)) + gdf = geopandas.read_file(input_path) # print(gdf) if gdf.empty: raise ValueError("Error: gdf is empty, No Labels found : Check your labels") gdf["geometry"] = gdf.apply(remove_self_intersection, axis=1) gdf.to_file(output_path, driver="GeoJSON") + + return diff --git a/hot_fair_utilities/preprocessing/multimasks_from_polygons.py b/hot_fair_utilities/preprocessing/multimasks_from_polygons.py index 6accf869..eb1b4aa4 100644 --- a/hot_fair_utilities/preprocessing/multimasks_from_polygons.py +++ b/hot_fair_utilities/preprocessing/multimasks_from_polygons.py @@ -6,12 +6,12 @@ # Third party imports import geopandas as gpd import rasterio as rio -from ramp.data_mgmt.chip_label_pairs import ( - construct_mask_filepath, - get_tq_chip_label_pairs, -) -from ramp.utils.img_utils import to_channels_first -from ramp.utils.multimask_utils import df_to_px_mask, multimask_to_sparse_multimask + +from ramp_fair.data_mgmt.chip_label_pairs import construct_mask_filepath +from ramp_fair.data_mgmt.chip_label_pairs import get_tq_chip_label_pairs + +from ramp_fair.utils.img_utils import to_channels_first +from ramp_fair.utils.multimask_utils import df_to_px_mask, multimask_to_sparse_multimask from solaris.utils.core import _check_rasterio_im_load from solaris.utils.geo import get_crs from solaris.vector.mask import crs_is_metric @@ -96,13 +96,23 @@ def multimasks_from_polygons( reference_im = _check_rasterio_im_load(chip_path) - if get_crs(gdf) != get_crs(reference_im): - # BUGFIX: if crs's don't match, reproject the geodataframe - gdf = gdf.to_crs(get_crs(reference_im)) + #check co-ordinate systems + gdf_crs = get_crs(gdf) + ref_crs = get_crs(reference_im) + + # Only attempt reprojection if both CRS values are valid and not equal. + if gdf_crs is None or ref_crs is None: + raise ValueError("Invalid CRS encountered in input data.") + if gdf_crs != ref_crs: + try: + gdf = gdf.to_crs(ref_crs) + print("Reprojected GeoDataFrame to reference CRS.") + except Exception as e: + print("Error reprojecting GeoDataFrame:", e) + raise if crs_is_metric(gdf): meters = True - boundary_width = min(reference_im.res) * input_boundary_width contact_spacing = min(reference_im.res) * input_contact_spacing diff --git a/hot_fair_utilities/preprocessing/preprocess.py b/hot_fair_utilities/preprocessing/preprocess.py index ea4c1469..4c221728 100644 --- a/hot_fair_utilities/preprocessing/preprocess.py +++ b/hot_fair_utilities/preprocessing/preprocess.py @@ -1,10 +1,10 @@ # Standard library imports import os -from ..georeferencing import georeference -from .clip_labels import clip_labels -from .fix_labels import fix_labels -from .reproject_labels import reproject_labels_to_epsg3857 +from hot_fair_utilities.georeferencing import georeference +from hot_fair_utilities.preprocessing.clip_labels import clip_labels +from hot_fair_utilities.preprocessing.fix_labels import fix_labels +from hot_fair_utilities.preprocessing.reproject_labels import reproject_labels_to_epsg3857 def preprocess( @@ -110,7 +110,7 @@ def preprocess( os.remove(f"{output_path}/labels_epsg3857.geojson") if multimasks: - from .multimasks_from_polygons import multimasks_from_polygons + from hot_fair_utilities.preprocessing.multimasks_from_polygons import multimasks_from_polygons assert os.path.isdir( f"{output_path}/chips" diff --git a/hot_fair_utilities/training/ramp/__init__.py b/hot_fair_utilities/training/ramp/__init__.py index c1d3fae6..5846e668 100644 --- a/hot_fair_utilities/training/ramp/__init__.py +++ b/hot_fair_utilities/training/ramp/__init__.py @@ -1 +1 @@ -from .train import train, run_feedback +from hot_fair_utilities.training.train import train, run_feedback diff --git a/hot_fair_utilities/training/ramp/run_training.py b/hot_fair_utilities/training/ramp/run_training.py index df76cebb..c7ea77bf 100644 --- a/hot_fair_utilities/training/ramp/run_training.py +++ b/hot_fair_utilities/training/ramp/run_training.py @@ -36,10 +36,12 @@ def __init__(self, message): # Third party imports import segmentation_models as sm + from ramp.data_mgmt.data_generator import ( test_batches_from_gtiff_dirs, training_batches_from_gtiff_dirs, ) + from ramp.training import ( callback_constructors, loss_constructors, @@ -52,14 +54,15 @@ def __init__(self, message): from ramp.training.augmentation_constructors import get_augmentation_fn from ramp.utils.misc_ramp_utils import get_num_files -from .config import RAMP_CONFIG +from hot_fair_utilities.training.ramp.config import RAMP_CONFIG # Segmentation Models: using `keras` framework. sm.set_framework("tf.keras") # this variable must be defined. It is the parent of the 'ramp-code' directory. -working_ramp_home = os.environ["RAMP_HOME"] +# working_ramp_home = os.environ["RAMP_HOME"] +repo_home = os.system("git rev-parse --show-toplevel") def apply_feedback( diff --git a/hot_fair_utilities/training/ramp/train.py b/hot_fair_utilities/training/ramp/train.py index 37ab1e34..b37694f0 100644 --- a/hot_fair_utilities/training/ramp/train.py +++ b/hot_fair_utilities/training/ramp/train.py @@ -1,9 +1,9 @@ # Standard library imports import os -from .cleanup import extract_highest_accuracy_model -from .prepare_data import split_training_2_validation -from .run_training import apply_feedback, manage_fine_tuning_config, run_main_train_code +from hot_fair_utilities.training.ramp.cleanup import extract_highest_accuracy_model +from hot_fair_utilities.training.ramp.prepare_data import split_training_2_validation +from hot_fair_utilities.training.ramp.run_training import apply_feedback, manage_fine_tuning_config, run_main_train_code def train( @@ -83,7 +83,7 @@ def run_feedback( assert os.path.exists(input_path), "Input Feedback Path Doesn't Exist" assert os.path.exists(feedback_base_model), "Feedback base Model Doesn't Exist" os.environ.update(os.environ) - os.environ["RAMP_HOME"] = model_home + # os.environ["RAMP_HOME"] = model_home print("Starting to prepare data for training") split_training_2_validation(input_path, output_path, multimasks) print("Data is ready for training") diff --git a/hot_fair_utilities/utils.py b/hot_fair_utilities/utils.py index 4ef7b388..225158c1 100644 --- a/hot_fair_utilities/utils.py +++ b/hot_fair_utilities/utils.py @@ -14,7 +14,6 @@ # Third party imports # Third-party imports import geopandas -import matplotlib.pyplot as plt import pandas as pd import requests import ultralytics @@ -257,7 +256,6 @@ def fetch_osm_data(payload: json, API_URL="https://api-prod.raw-data.hotosm.org/ # Third party imports -import matplotlib.pyplot as plt import pandas as pd diff --git a/requirements.txt b/requirements.txt index e629c901..51111778 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ black isort build twine +protobuf==3.19.6 # torch==1.12.1 # torchvision==0.13.1 # ultralytics==8.1.6 diff --git a/setup.py b/setup.py index 2c008f27..3f850953 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,10 @@ -# Third party imports -from setuptools import find_packages, setup +from setuptools import setup, find_packages +import os + +if os.path.exists('./ramp-code'): + pass +else: + print('[WARNING] - ramp-code folder does not exist, some functionalities may not work correctly.') setup( package_dir={"": "."}, diff --git a/test_ramp.py b/test_ramp.py index 465e9034..7ecfb2e5 100644 --- a/test_ramp.py +++ b/test_ramp.py @@ -9,12 +9,13 @@ print( f"\nUsing tensorflow version {tf. __version__} with no of gpu : {len(tf.config.experimental.list_physical_devices('GPU'))}\n" ) -print(os.getcwd()) -os.environ.update(os.environ) +# print(os.getcwd()) +# os.environ.update(os.environ) + # Add a new environment variable to the operating system -os.environ["RAMP_HOME"] = os.getcwd() +# os.environ["RAMP_HOME"] = os.getcwd() # Print the environment variables to verify that the new variable was added -print(os.environ["RAMP_HOME"]) +# print(os.environ["RAMP_HOME"]) start_time = time.time() # Third party imports @@ -22,38 +23,67 @@ # import ramp.utils # Reader imports -import hot_fair_utilities - -base_path = f"{os.getcwd()}/ramp-data/sample_2" +# import hot_fair_utilities +top_level_repo_folder_path = str(os.system("git rev-parse --show-toplevel")) +sample_ramp_data_folder_path = os.path.join(top_level_repo_folder_path,'ramp-data/sample_2') # Reader imports # %% from hot_fair_utilities import preprocess -model_input_image_path = f"{base_path}/input" -preprocess_output = f"/{base_path}/preprocessed" -preprocess( - input_path=model_input_image_path, - output_path=preprocess_output, - rasterize=True, - rasterize_options=["binary"], - georeference_images=True, - multimasks=True, -) +ramp_sample_input_folder = os.path.join(sample_ramp_data_folder_path,'input') +# model_input_image_path = f"{base_path}/input" +ramp_sample_preprocessed_folder = os.path.join(sample_ramp_data_folder_path,'preprocessed') +# preprocess_output = f"/{base_path}/preprocessed" + +def test_preprocess_binary(): + preprocess( + input_path=ramp_sample_input_folder, + output_path=ramp_sample_preprocessed_folder, + rasterize=True, + rasterize_options=["binary"], + georeference_images=True, + multimasks=False + ) + print("Preprocessing with binary masks completed successfully.") + + return + +test_preprocess_binary() + +def test_preprocess_multimask(): + preprocess( + input_path=ramp_sample_input_folder, + output_path=ramp_sample_preprocessed_folder, + rasterize=True, + rasterize_options=["binary"], + georeference_images=True, + multimasks=True + ) + + print("Preprocessing with multi-masks completed successfully.") + + return + +test_preprocess_multimask() # Reader imports # %% from hot_fair_utilities.training.ramp import train # %% -train_output = f"{base_path}/train" +# train_output = f"{base_path}/train" +ramp_sample_ready_for_training_folder = os.path.join(sample_ramp_data_folder_path,'ready_for_trainining') + + final_accuracy, final_model_path = train( - input_path=preprocess_output, - output_path=train_output, + input_path=ramp_sample_preprocessed_folder, + output_path=ramp_sample_ready_for_training_folder, epoch_size=2, batch_size=2, model="ramp", - model_home=os.environ["RAMP_HOME"], + # model_home=os.environ["RAMP_HOME"], + model_home=top_level_repo_folder_path ) # %% @@ -63,10 +93,13 @@ # %% from hot_fair_utilities import predict -prediction_output = f"{base_path}/prediction/output" +# prediction_output = f"{base_path}/prediction/output" +prediction_input = os.path.join(sample_ramp_data_folder_path,'prediction/input') +prediiction_output = os.path.join(sample_ramp_data_folder_path,'prediction/output') + predict( checkpoint_path=final_model_path, - input_path=f"{base_path}/prediction/input", + input_path=prediction_input, prediction_path=prediction_output, )