From 715e2cf2f9b14c85f85236008a2dba4bf0a0c617 Mon Sep 17 00:00:00 2001 From: Glenn Fung <117829736+glenntfung@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:23:01 -0500 Subject: [PATCH 1/5] FIX: region outflow #204 If the user annotates a region outside of the image, the downloaded data will only reflect portions within the image. --- server/app.py | 194 ++++++++++++++++++++++++++++---------------------- 1 file changed, 109 insertions(+), 85 deletions(-) diff --git a/server/app.py b/server/app.py index 2361978..a40d8a7 100644 --- a/server/app.py +++ b/server/app.py @@ -82,6 +82,63 @@ def save_settings(settings): json.dump(settings, f, indent=4) +def clamp_region_coordinates(region): + """Clamps the coordinates of a region to be within image bounds [0.0, 1.0]. + This version is robust to handle data formats from the database.""" + + def _parse_coord(v): + """Helper to parse coordinate values that might be strings or lists.""" + if isinstance(v, str) and v.startswith("[") and v.endswith("]"): + return float(v[1:-1]) + if isinstance(v, list) and len(v) > 0: + return float(v[0]) + # Return the value as is if it doesn't match the above conditions, + # it will be converted to float in _clamp_value. + return v + + def _clamp_value(v): + """Helper to clamp a single float value after parsing.""" + try: + # Parse first, then convert to float and clamp + val = float(_parse_coord(v)) + return max(0.0, min(val, 1.0)) + except (ValueError, TypeError): + return 0.0 # Default to 0.0 if parsing fails + + region_type = region.get("type") + + if region_type == "polygon" and "points" in region and region["points"]: + clamped_points = [ + [_clamp_value(p[0]), _clamp_value(p[1])] for p in region["points"] + ] + region["points"] = clamped_points + + elif region_type == "box" and all(k in region for k in ("x", "y", "w", "h")): + x = _clamp_value(region["x"]) + y = _clamp_value(region["y"]) + # Width and height are deltas, so we just parse them, not clamp them to 1.0 + w = float(_parse_coord(region["w"])) + h = float(_parse_coord(region["h"])) + + if x + w > 1.0: w = 1.0 - x + if y + h > 1.0: h = 1.0 - y + + region.update({"x": x, "y": y, "w": w, "h": h}) + + elif region_type == "circle" and all(k in region for k in ("rx", "ry", "rw", "rh")): + rx = _clamp_value(region["rx"]) + ry = _clamp_value(region["ry"]) + rw = float(_parse_coord(region["rw"])) + rh = float(_parse_coord(region["rh"])) + + if rx + rw > 1.0: rw = 1.0 - rx + if ry + rh > 1.0: rh = 1.0 - ry + + region.update({"rx": rx, "ry": ry, "rw": rw, "rh": rh}) + + return region + + dbModule = Module() path = os.path.abspath("./uploads") @@ -91,6 +148,12 @@ def save_settings(settings): def save_annotate_info(): try: request_data = request.get_json() + + if "regions" in request_data and isinstance(request_data["regions"], list): + request_data["regions"] = [ + clamp_region_coordinates(r) for r in request_data["regions"] + ] + if dbModule.handleNewData(request_data): # Return success response return ( @@ -282,6 +345,12 @@ def delete_file(filename): def save_active_image_info(): try: request_data = request.get_json() + + if "regions" in request_data and isinstance(request_data["regions"], list): + request_data["regions"] = [ + clamp_region_coordinates(r) for r in request_data["regions"] + ] + # Assume handleActiveImageData returns True if successful if dbModule.handleActiveImageData(request_data): # Return success response @@ -370,7 +439,8 @@ def add_regions(regions, region_type=None): ] region["points"] = decoded_points region["type"] = region_type - main_dict["regions"].append(region) + clamped_region = clamp_region_coordinates(region) + main_dict["regions"].append(clamped_region) if polygonRegions is not None: add_regions(polygonRegions, "polygon") @@ -1023,26 +1093,26 @@ def create_yolo_annotations(image_names, color_map=None): # Process polygon regions if polygonRegions is not None: - for index, region in polygonRegions.iterrows(): - class_name = region.get("class", "unknown") + for index, region_series in polygonRegions.iterrows(): + region = region_series.to_dict() + region['type'] = 'polygon' + # Decode points string points_str = region.get("points", "") - - # Split points string into individual points - points_list = points_str.split(";") - - # Convert points to list of tuples - points = [] - for point_str in points_list: - x, y = map(float, point_str.split("-")) - points.append((x, y)) - + if points_str: + region['points'] = [[float(c) for c in p.split('-')] for p in points_str.split(';')] + else: + region['points'] = [] + + clamped_region = clamp_region_coordinates(region) + clamped_points = clamped_region.get('points', []) + # Convert points to normalized YOLO format - if points: - xmin = min(point[0] for point in points) - ymin = min(point[1] for point in points) - xmax = max(point[0] for point in points) - ymax = max(point[1] for point in points) - + if clamped_points: + class_name = clamped_region.get("class", "unknown") + xmin = min(p[0] for p in clamped_points) + ymin = min(p[1] for p in clamped_points) + xmax = max(p[0] for p in clamped_points) + ymax = max(p[1] for p in clamped_points) # YOLO format: class_index x_center y_center width height (all normalized) annotations.append( f"{class_name} {(xmin + xmax) / 2:.6f} {(ymin + ymax) / 2:.6f} {xmax - xmin:.6f} {ymax - ymin:.6f}" @@ -1050,33 +1120,13 @@ def create_yolo_annotations(image_names, color_map=None): # Process box regions if boxRegions is not None: - for index, region in boxRegions.iterrows(): - class_name = region.get("class", "unknown") - try: - x = ( - float(region["x"][1:-1]) - if isinstance(region["x"], str) - else float(region["x"][0]) - ) - y = ( - float(region["y"][1:-1]) - if isinstance(region["y"], str) - else float(region["y"][0]) - ) - w = ( - float(region["w"][1:-1]) - if isinstance(region["w"], str) - else float(region["w"][0]) - ) - h = ( - float(region["h"][1:-1]) - if isinstance(region["h"], str) - else float(region["h"][0]) - ) - except (ValueError, TypeError) as e: - raise ValueError( - f"Invalid format in region dimensions: {region}, Error: {e}" - ) + for index, region_series in boxRegions.iterrows(): + region = region_series.to_dict() + region['type'] = 'box' + clamped_region = clamp_region_coordinates(region) + + class_name = clamped_region.get("class", "unknown") + x, y, w, h = clamped_region['x'], clamped_region['y'], clamped_region['w'], clamped_region['h'] # YOLO format: class_index x_center y_center width height (all normalized) annotations.append( f"{class_name} {x + w / 2:.6f} {y + h / 2:.6f} {w:.6f} {h:.6f}" @@ -1084,44 +1134,18 @@ def create_yolo_annotations(image_names, color_map=None): # Process circle/ellipse regions if circleRegions is not None: - for index, region in circleRegions.iterrows(): - class_name = region.get("class", "unknown") - try: - rx = ( - float(region["rx"][1:-1]) * width - if isinstance(region["rx"], str) - else float(region["rx"][0]) - ) - ry = ( - float(region["ry"][1:-1]) * height - if isinstance(region["ry"], str) - else float(region["ry"][0]) - ) - rw = ( - float(region["rw"][1:-1]) * width - if isinstance(region["rw"], str) - else float(region["rw"][0]) - ) - rh = ( - float(region["rh"][1:-1]) * height - if isinstance(region["rh"], str) - else float(region["rh"][0]) - ) - except (ValueError, TypeError) as e: - raise ValueError( - f"Invalid format in region dimensions: {region}, Error: {e}" - ) - - # For YOLO, if width and height are equal, it represents a circle - if rw == rh: - annotations.append( - f"{class_name} {rx:.6f} {ry:.6f} {rw:.6f} {rw:.6f}" - ) # Treat as circle - else: - # Treat as ellipse (YOLO does not directly support ellipse, so treat as box) - annotations.append( - f"{class_name} {rx + rw / 2:.6f} {ry + rh / 2:.6f} {rw:.6f} {rh:.6f}" - ) + for index, region_series in circleRegions.iterrows(): + region = region_series.to_dict() + region['type'] = 'circle' + clamped_region = clamp_region_coordinates(region) + + class_name = clamped_region.get("class", "unknown") + rx, ry, rw, rh = clamped_region['rx'], clamped_region['ry'], clamped_region['rw'], clamped_region['rh'] + + # Treat circle/ellipse as a bounding box for YOLO format (YOLO does not directly support ellipses) + annotations.append( + f"{class_name} {rx + rw / 2:.6f} {ry + rh / 2:.6f} {rw:.6f} {rh:.6f}" + ) # Append annotations for current image to all_annotations list all_annotations.extend(annotations) @@ -1247,4 +1271,4 @@ def main(): # If the file is run directly,start the app. if __name__ == "__main__": print("Starting server...") - app.run(debug=False) + app.run(debug=False) \ No newline at end of file From 21217d347ad2cc5d2f194be5d4421962e812f06a Mon Sep 17 00:00:00 2001 From: Glenn Fung <117829736+glenntfung@users.noreply.github.com> Date: Sun, 31 Aug 2025 01:27:06 -0500 Subject: [PATCH 2/5] FIX(Client): No annotator out of image --- client/src/ImageCanvas/use-mouse.js | 30 +++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/client/src/ImageCanvas/use-mouse.js b/client/src/ImageCanvas/use-mouse.js index c357b9f..4ae14ab 100644 --- a/client/src/ImageCanvas/use-mouse.js +++ b/client/src/ImageCanvas/use-mouse.js @@ -1,6 +1,8 @@ // @flow weak import { useRef } from "react" +const clamp = (n, lo, hi) => (n < lo ? lo : n > hi ? hi : n) +const isInside01 = (x, y) => x >= 0 && x <= 1 && y >= 0 && y <= 1 import { Matrix } from "transformation-matrix-js" const getDefaultMat = () => Matrix.from(1, 0, 0, 1, -10, -10) @@ -23,6 +25,7 @@ export default ({ dragging, }) => { const mousePosition = useRef({ x: 0, y: 0 }) + const insideRef = useRef(true) // tracks if pointer is inside the rendered image const prevMousePosition = useRef({ x: 0, y: 0 }) const zoomIn = (direction, point) => { @@ -57,7 +60,16 @@ export default ({ } const { iw, ih } = layoutParams.current - onMouseMove({ x: projMouse.x / iw, y: projMouse.y / ih }) + // onMouseMove({ x: projMouse.x / iw, y: projMouse.y / ih }) + const nx = projMouse.x / iw + const ny = projMouse.y / ih + const inside = isInside01(nx, ny) + insideRef.current = inside + + // Only notify annotators while the pointer is over the image + if (inside) { + onMouseMove({ x: nx, y: ny }) + } if (dragging) { mat.translate( @@ -89,14 +101,20 @@ export default ({ return } if (e.button === 0) { + const { iw, ih } = layoutParams.current + const nx = projMouse.x / iw + const ny = projMouse.y / ih + // Block starting any annotation outside the image + if (!isInside01(nx, ny)) return if (specialEvent.type === "resize-box") { // onResizeBox() } if (specialEvent.type === "move-region") { // onResizeBox() } - const { iw, ih } = layoutParams.current - onMouseDown({ x: projMouse.x / iw, y: projMouse.y / ih }) + // const { iw, ih } = layoutParams.current + // onMouseDown({ x: projMouse.x / iw, y: projMouse.y / ih }) + onMouseDown({ x: nx, y: ny }) } }, onMouseUp: (e) => { @@ -152,7 +170,11 @@ export default ({ return changeDragging(false) if (e.button === 0) { const { iw, ih } = layoutParams.current - onMouseUp({ x: projMouse.x / iw, y: projMouse.y / ih }) + // onMouseUp({ x: projMouse.x / iw, y: projMouse.y / ih }) + // Clamp release to the image edge so we never create/update out of bounds + const nx = clamp(projMouse.x / iw, 0, 1) + const ny = clamp(projMouse.y / ih, 0, 1) + onMouseUp({ x: nx, y: ny }) } }, onWheel: (e) => { From 9bfd94a8154747898554b82f30b09a54cf1d4315 Mon Sep 17 00:00:00 2001 From: Glenn Fung <117829736+glenntfung@users.noreply.github.com> Date: Sun, 31 Aug 2025 17:23:47 -0500 Subject: [PATCH 3/5] Revert server change --- server/app.py | 192 ++++++++++++++++++++++---------------------------- 1 file changed, 84 insertions(+), 108 deletions(-) diff --git a/server/app.py b/server/app.py index a40d8a7..a7a9fd3 100644 --- a/server/app.py +++ b/server/app.py @@ -82,63 +82,6 @@ def save_settings(settings): json.dump(settings, f, indent=4) -def clamp_region_coordinates(region): - """Clamps the coordinates of a region to be within image bounds [0.0, 1.0]. - This version is robust to handle data formats from the database.""" - - def _parse_coord(v): - """Helper to parse coordinate values that might be strings or lists.""" - if isinstance(v, str) and v.startswith("[") and v.endswith("]"): - return float(v[1:-1]) - if isinstance(v, list) and len(v) > 0: - return float(v[0]) - # Return the value as is if it doesn't match the above conditions, - # it will be converted to float in _clamp_value. - return v - - def _clamp_value(v): - """Helper to clamp a single float value after parsing.""" - try: - # Parse first, then convert to float and clamp - val = float(_parse_coord(v)) - return max(0.0, min(val, 1.0)) - except (ValueError, TypeError): - return 0.0 # Default to 0.0 if parsing fails - - region_type = region.get("type") - - if region_type == "polygon" and "points" in region and region["points"]: - clamped_points = [ - [_clamp_value(p[0]), _clamp_value(p[1])] for p in region["points"] - ] - region["points"] = clamped_points - - elif region_type == "box" and all(k in region for k in ("x", "y", "w", "h")): - x = _clamp_value(region["x"]) - y = _clamp_value(region["y"]) - # Width and height are deltas, so we just parse them, not clamp them to 1.0 - w = float(_parse_coord(region["w"])) - h = float(_parse_coord(region["h"])) - - if x + w > 1.0: w = 1.0 - x - if y + h > 1.0: h = 1.0 - y - - region.update({"x": x, "y": y, "w": w, "h": h}) - - elif region_type == "circle" and all(k in region for k in ("rx", "ry", "rw", "rh")): - rx = _clamp_value(region["rx"]) - ry = _clamp_value(region["ry"]) - rw = float(_parse_coord(region["rw"])) - rh = float(_parse_coord(region["rh"])) - - if rx + rw > 1.0: rw = 1.0 - rx - if ry + rh > 1.0: rh = 1.0 - ry - - region.update({"rx": rx, "ry": ry, "rw": rw, "rh": rh}) - - return region - - dbModule = Module() path = os.path.abspath("./uploads") @@ -148,12 +91,6 @@ def _clamp_value(v): def save_annotate_info(): try: request_data = request.get_json() - - if "regions" in request_data and isinstance(request_data["regions"], list): - request_data["regions"] = [ - clamp_region_coordinates(r) for r in request_data["regions"] - ] - if dbModule.handleNewData(request_data): # Return success response return ( @@ -345,12 +282,6 @@ def delete_file(filename): def save_active_image_info(): try: request_data = request.get_json() - - if "regions" in request_data and isinstance(request_data["regions"], list): - request_data["regions"] = [ - clamp_region_coordinates(r) for r in request_data["regions"] - ] - # Assume handleActiveImageData returns True if successful if dbModule.handleActiveImageData(request_data): # Return success response @@ -439,8 +370,7 @@ def add_regions(regions, region_type=None): ] region["points"] = decoded_points region["type"] = region_type - clamped_region = clamp_region_coordinates(region) - main_dict["regions"].append(clamped_region) + main_dict["regions"].append(region) if polygonRegions is not None: add_regions(polygonRegions, "polygon") @@ -1093,26 +1023,26 @@ def create_yolo_annotations(image_names, color_map=None): # Process polygon regions if polygonRegions is not None: - for index, region_series in polygonRegions.iterrows(): - region = region_series.to_dict() - region['type'] = 'polygon' - # Decode points string + for index, region in polygonRegions.iterrows(): + class_name = region.get("class", "unknown") points_str = region.get("points", "") - if points_str: - region['points'] = [[float(c) for c in p.split('-')] for p in points_str.split(';')] - else: - region['points'] = [] - - clamped_region = clamp_region_coordinates(region) - clamped_points = clamped_region.get('points', []) - + + # Split points string into individual points + points_list = points_str.split(";") + + # Convert points to list of tuples + points = [] + for point_str in points_list: + x, y = map(float, point_str.split("-")) + points.append((x, y)) + # Convert points to normalized YOLO format - if clamped_points: - class_name = clamped_region.get("class", "unknown") - xmin = min(p[0] for p in clamped_points) - ymin = min(p[1] for p in clamped_points) - xmax = max(p[0] for p in clamped_points) - ymax = max(p[1] for p in clamped_points) + if points: + xmin = min(point[0] for point in points) + ymin = min(point[1] for point in points) + xmax = max(point[0] for point in points) + ymax = max(point[1] for point in points) + # YOLO format: class_index x_center y_center width height (all normalized) annotations.append( f"{class_name} {(xmin + xmax) / 2:.6f} {(ymin + ymax) / 2:.6f} {xmax - xmin:.6f} {ymax - ymin:.6f}" @@ -1120,13 +1050,33 @@ def create_yolo_annotations(image_names, color_map=None): # Process box regions if boxRegions is not None: - for index, region_series in boxRegions.iterrows(): - region = region_series.to_dict() - region['type'] = 'box' - clamped_region = clamp_region_coordinates(region) - - class_name = clamped_region.get("class", "unknown") - x, y, w, h = clamped_region['x'], clamped_region['y'], clamped_region['w'], clamped_region['h'] + for index, region in boxRegions.iterrows(): + class_name = region.get("class", "unknown") + try: + x = ( + float(region["x"][1:-1]) + if isinstance(region["x"], str) + else float(region["x"][0]) + ) + y = ( + float(region["y"][1:-1]) + if isinstance(region["y"], str) + else float(region["y"][0]) + ) + w = ( + float(region["w"][1:-1]) + if isinstance(region["w"], str) + else float(region["w"][0]) + ) + h = ( + float(region["h"][1:-1]) + if isinstance(region["h"], str) + else float(region["h"][0]) + ) + except (ValueError, TypeError) as e: + raise ValueError( + f"Invalid format in region dimensions: {region}, Error: {e}" + ) # YOLO format: class_index x_center y_center width height (all normalized) annotations.append( f"{class_name} {x + w / 2:.6f} {y + h / 2:.6f} {w:.6f} {h:.6f}" @@ -1134,18 +1084,44 @@ def create_yolo_annotations(image_names, color_map=None): # Process circle/ellipse regions if circleRegions is not None: - for index, region_series in circleRegions.iterrows(): - region = region_series.to_dict() - region['type'] = 'circle' - clamped_region = clamp_region_coordinates(region) - - class_name = clamped_region.get("class", "unknown") - rx, ry, rw, rh = clamped_region['rx'], clamped_region['ry'], clamped_region['rw'], clamped_region['rh'] - - # Treat circle/ellipse as a bounding box for YOLO format (YOLO does not directly support ellipses) - annotations.append( - f"{class_name} {rx + rw / 2:.6f} {ry + rh / 2:.6f} {rw:.6f} {rh:.6f}" - ) + for index, region in circleRegions.iterrows(): + class_name = region.get("class", "unknown") + try: + rx = ( + float(region["rx"][1:-1]) * width + if isinstance(region["rx"], str) + else float(region["rx"][0]) + ) + ry = ( + float(region["ry"][1:-1]) * height + if isinstance(region["ry"], str) + else float(region["ry"][0]) + ) + rw = ( + float(region["rw"][1:-1]) * width + if isinstance(region["rw"], str) + else float(region["rw"][0]) + ) + rh = ( + float(region["rh"][1:-1]) * height + if isinstance(region["rh"], str) + else float(region["rh"][0]) + ) + except (ValueError, TypeError) as e: + raise ValueError( + f"Invalid format in region dimensions: {region}, Error: {e}" + ) + + # For YOLO, if width and height are equal, it represents a circle + if rw == rh: + annotations.append( + f"{class_name} {rx:.6f} {ry:.6f} {rw:.6f} {rw:.6f}" + ) # Treat as circle + else: + # Treat as ellipse (YOLO does not directly support ellipse, so treat as box) + annotations.append( + f"{class_name} {rx + rw / 2:.6f} {ry + rh / 2:.6f} {rw:.6f} {rh:.6f}" + ) # Append annotations for current image to all_annotations list all_annotations.extend(annotations) From e1a27a62e8db51634aa13d3a2fe1fbf7b1797a3c Mon Sep 17 00:00:00 2001 From: Glenn Fung <117829736+glenntfung@users.noreply.github.com> Date: Sun, 31 Aug 2025 18:00:15 -0500 Subject: [PATCH 4/5] FIX(server): safe image mode --- server/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/app.py b/server/app.py index a7a9fd3..ff5a2d4 100644 --- a/server/app.py +++ b/server/app.py @@ -544,7 +544,7 @@ def download_image_with_annotations(): ) response = requests.get(image_url) - image = Image.open(BytesIO(response.content)) + image = Image.open(BytesIO(response.content)).convert("RGBA") draw = ImageDraw.Draw(image) for region in image_info.get("regions", []): @@ -852,7 +852,7 @@ def download_image_mask(): response = requests.get(image_url) response.raise_for_status() - image = Image.open(BytesIO(response.content)) + image = Image.open(BytesIO(response.content)).convert("RGBA") width, height = image.size mask = Image.new( "RGB", (width, height), app.config["MASK_BACKGROUND_COLOR"] From bfdd9ea272b01fa9772bd717e490f489b6f6c162 Mon Sep 17 00:00:00 2001 From: Glenn Fung <117829736+glenntfung@users.noreply.github.com> Date: Sun, 31 Aug 2025 18:07:39 -0500 Subject: [PATCH 5/5] FIX(server): check before convert --- server/app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/app.py b/server/app.py index ff5a2d4..7e63a0a 100644 --- a/server/app.py +++ b/server/app.py @@ -544,7 +544,9 @@ def download_image_with_annotations(): ) response = requests.get(image_url) - image = Image.open(BytesIO(response.content)).convert("RGBA") + image = Image.open(BytesIO(response.content)) + if image.mode != "RGBA": + image = image.convert("RGBA") draw = ImageDraw.Draw(image) for region in image_info.get("regions", []): @@ -852,7 +854,9 @@ def download_image_mask(): response = requests.get(image_url) response.raise_for_status() - image = Image.open(BytesIO(response.content)).convert("RGBA") + image = Image.open(BytesIO(response.content)) + if image.mode != "RGBA": + image = image.convert("RGBA") width, height = image.size mask = Image.new( "RGB", (width, height), app.config["MASK_BACKGROUND_COLOR"]