FIX: onbedekt consolidation category (#206)

Author: lvjoebro

cropclassification/calc_cover.py

@@ -49,6 +49,7 @@ def run_cover(
     logger.warning("This is a POC for a cover marker, so not for operational use!")
     logger.info(f"Config used: \n{conf.pformat_config()}")
 
+    force = True
     markertype = conf.marker["markertype"]
     if not markertype.startswith(("COVER", "ONBEDEKT")):
         raise ValueError(f"Invalid markertype {markertype}, expected COVER_XXX")
@@ -88,7 +89,9 @@ def run_cover(
             OR "ALL_BEST" like '%TBG%'
             OR "ALL_BEST" like '%EBG%'
         """
-        gfo.copy_layer(input_parcel_path, input_parcel_filtered_path, where=where)
+        gfo.copy_layer(
+            input_parcel_path, input_parcel_filtered_path, where=where, force=force
+        )
         input_parcel_path = input_parcel_filtered_path
 
     elif markertype == "COVER_EEB_VOORJAAR":
@@ -100,7 +103,9 @@ def run_cover(
         where = (
             "ALL_BEST like '%EEB%' AND GWSCOD_V = '83' AND GWSCOD_H IN ('201', '202')"
         )
-        gfo.copy_layer(input_parcel_path, input_parcel_filtered_path, where=where)
+        gfo.copy_layer(
+            input_parcel_path, input_parcel_filtered_path, where=where, force=force
+        )
         input_parcel_path = input_parcel_filtered_path
 
     elif markertype in ("COVER_TBG_BMG_VOORJAAR", "COVER_TBG_BMG_NAJAAR"):
@@ -110,7 +115,9 @@ def run_cover(
         input_parcel_filtered_path = input_preprocessed_dir / input_parcel_filename
 
         where = "ALL_BEST like '%TBG%' OR ALL_BEST like '%BMG%'"
-        gfo.copy_layer(input_parcel_path, input_parcel_filtered_path, where=where)
+        gfo.copy_layer(
+            input_parcel_path, input_parcel_filtered_path, where=where, force=force
+        )
         input_parcel_path = input_parcel_filtered_path
 
     else:
@@ -144,6 +151,7 @@ def run_cover(
         input_parcel_path=input_parcel_path,
         output_imagedata_parcel_input_path=imagedata_input_parcel_path,
         output_parcel_nogeo_path=input_parcel_nogeo_path,
+        force=force,
     )
 
     # STEP 2: Calculate the timeseries data needed
@@ -165,6 +173,7 @@ def run_cover(
         end_date=end_date,
         images_to_use=images_to_use,
         timeseries_periodic_dir=timeseries_periodic_dir,
+        force=force,
     )
 
     # STEP 3: Determine the cover for the parcels for all periods
@@ -175,7 +184,7 @@ def run_cover(
     )
     cover_dir = run_dir / input_parcel_nogeo_path.stem
     cover_dir.mkdir(parents=True, exist_ok=True)
-    force = False
+
     on_error = "warn"
     parcels_cover_paths = []
 
@@ -238,10 +247,12 @@ def run_cover(
     cols_to_keep = [*list(input_info.columns), "pred1"]
     if "provincie" in parcels_selected.columns:
         cols_to_keep.append("provincie")
+
     parcels_selected = (
         parcels_selected[[*cols_to_keep, "pred1_prob"]]
-        .groupby(cols_to_keep, dropna=False, as_index=False)
-        .max()
+        .sort_values("pred1_prob", ascending=False)
+        .groupby(conf.columns["id"], dropna=False, as_index=False)
+        .first()  # Take the highest pred1_prob per id
     )
 
     # Add pred_consolidated based on max pred1_proba
@@ -268,7 +279,7 @@ def _categorize_pred(x):
         return "NODATA"
     try:
         x_num = float(x)
-        if x_num >= 0.5:
+        if x_num > 0.5:
             return "ONBEDEKT"
         elif x_num > 0.4:
             return "DOUBT"
@@ -295,7 +306,7 @@ def _calc_cover(
         if force:
             gfo.remove(output_path)
         elif output_geo_path is not None and not output_geo_path.exists():
-            # Geo file is asked bu missing: we need to recalculate the output file as
+            # Geo file is asked but missing: we need to recalculate the output file as
             # well because we need temporary files to create the geo file.
             gfo.remove(output_path)
         else:

cropclassification/preprocess/_timeseries_calc_openeo.py

@@ -27,6 +27,7 @@ def calculate_periodic_timeseries(
     timeseries_periodic_dir: Path,
     nb_parallel: int,
     on_missing_image: str,
+    force: bool = False,
 ):
     """Calculate timeseries data for the input parcels.
 
@@ -49,6 +50,8 @@ def calculate_periodic_timeseries(
             - ignore: ignore that the image, don't try to download it
             - calculate_raise: calculate the image and raise an error if it fails
             - calculate_ignore: calculate the image and ignore the error if it fails
+        force (bool = False): whether to force recalculation of existing data.
+            (will not redownload images)
     """
     info = gfo.get_layerinfo(input_parcel_path)
     if info.crs is not None and not info.crs.equals(roi_crs):
@@ -67,7 +70,7 @@ def calculate_periodic_timeseries(
         imageprofiles_to_get=imageprofiles_to_get,
         imageprofiles=imageprofiles,
         on_missing_image=on_missing_image,
-        force=False,
+        force=False,  # dont redownload on force
     )
 
     # Now calculate the timeseries
@@ -94,4 +97,5 @@ def calculate_periodic_timeseries(
         stats=["count", "mean", "median", "std", "min", "max"],
         engine="pyqgis",
         nb_parallel=nb_parallel,
+        force=force,
     )

cropclassification/preprocess/_timeseries_helper.py

@@ -84,6 +84,9 @@ def prepare_input(
     if output_parcel_nogeo_path is not None and (
         force is True or not output_parcel_nogeo_path.exists()
     ):
+        if output_parcel_nogeo_path.exists():
+            os.remove(output_parcel_nogeo_path)
+
         logger.info(f"Save non-geo data to {output_parcel_nogeo_path}")
         parceldata_nogeo_df = parceldata_gdf.drop(["geometry"], axis=1)
         pdh.to_file(parceldata_nogeo_df, output_parcel_nogeo_path)

cropclassification/preprocess/timeseries.py

@@ -27,6 +27,7 @@ def calc_timeseries_data(
     end_date: datetime,
     images_to_use: dict[str, conf.ImageConfig],
     timeseries_periodic_dir: Path,
+    force: bool = False,
 ):
     """Calculate timeseries data for the input parcels.
 
@@ -40,6 +41,7 @@ def calc_timeseries_data(
         images_to_use (List[str]): an array with data you want to be calculated:
             check out the constants starting with DATA_TO_GET... for the options.
         timeseries_periodic_dir (Path): Directory the timeseries will be written to.
+        force (bool = False): whether to force recalculation of existing data.
     """
     # Check some variables...
     if images_to_use is None:
@@ -65,6 +67,7 @@ def calc_timeseries_data(
             timeseries_periodic_dir=timeseries_periodic_dir,
             nb_parallel=conf.general.getint("nb_parallel", -1),
             on_missing_image=conf.images.get("on_missing_image", "calculate_raise"),
+            force=force,
         )