Updating master and background-rm (includes updated README)

.gitignore

@@ -4,6 +4,7 @@
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
+build/
 .ipynb_checkpoints/
 *.py[cod]
 *$py.class
@@ -14,9 +15,11 @@ __pycache__/
 
 # C extensions
 *.so
+*.o
 
 # IDE files
 .idea
 .vs_code/
+.vscode/
 
 data/

README.md

@@ -13,20 +13,25 @@ activate it with
 
     conda activate urobotics
 
-and install all dependencies with
-
-    pip3 install -r requirements.txt
-
 Then clone the repo in a directory of your choice
 
     git clone https://github.com/berkeleyauv/perception.git
 
-and install it
+and change directory to the cloned directory and install all dependencies with
+
+    pip3 install -r requirements.txt
+
+Then install the perception module with
 
     pip3 install -e perception/
 
 Also, our training data is stored here https://www.dropbox.com/sh/rrbfqfutrmifrxs/AAAfXxlcCtWZmUELp4wXyTIxa?dl=0 so download it and unzip it in the same folder as `perception`.
 
+### Cython
+To compile cythonized code, run the following commands after `cd`ing into the folder with Cython `setup.py` (`perception/tasks/segmentation/saliency_detection`)
+
+    python setup.py build_ext --inplace
+
 
 ## misc:
 Misc code, camera calibration etc.

misc/general_bg_rm.py

@@ -0,0 +1,84 @@
+import cv2 as cv
+import numpy as np
+import os
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument("file", help="file")
+args = parser.parse_args()
+from dark_channel.handler import process_frame as dark_channel
+
+
+
+def DarkChannel(im):
+    b,g,r = cv.split(im)
+    dc = cv.min(cv.min(r,g),b);
+    kernel = cv.getStructuringElement(cv.MORPH_RECT,(im.shape[0],im.shape[1]))
+    dark = cv.erode(dc,kernel)
+    return dark
+
+def resize_frame(frame,ratio = 0.4):
+    return cv.resize(frame,(int(frame.shape[1]*ratio),int(frame.shape[0]*ratio)))
+
+def save_frames(frames,folder):
+    os.mkdir(folder)
+    [cv.imwrite(f'{folder}/{frame}.png', frames[frame]) for frame in frames]
+
+def show_frames(frames):
+    [cv.imshow(frame,frames[frame]) for frame in frames]
+
+def analyze(src):
+
+    # src = cv.imread(fn);
+    # src = resize_frame(src)
+    img = src
+    gray = cv.cvtColor(img,cv.COLOR_BGR2GRAY)
+    ret, thresh = cv.threshold(gray,0,255,cv.THRESH_BINARY_INV+cv.THRESH_OTSU)
+
+        # noise removal
+    kernel = np.ones((3,3),np.uint8)
+    opening = cv.morphologyEx(thresh,cv.MORPH_OPEN,kernel, iterations = 2)
+    # # sure background area
+    # sure_bg = cv.dilate(opening,kernel,iterations=3)
+    # # Finding sure foreground area
+    # dist_transform = cv.distanceTransform(opening,cv.DIST_L2,5)
+    # ret, sure_fg = cv.threshold(dist_transform,0.7*dist_transform.max(),255,0)
+    # # Finding unknown region
+    # sure_fg = np.uint8(sure_fg)
+    # unknown = cv.subtract(sure_bg,sure_fg)
+
+    # # Marker labelling
+    # ret, markers = cv.connectedComponents(sure_fg)
+    # # Add one to all labels so that sure background is not 0, but 1
+    # markers = markers+1
+    # # Now, mark the region of unknown with zero
+    # markers[unknown==255] = 0
+    # markers = cv.watershed(img,markers)
+    # orig = resize_frame(cv.imread(fn))
+    # orig[markers == -1] = [255,0,0]
+    return thresh
+    # frames = {'test':thresh,'dist':dist_transform,'sure_fg':sure_fg,'sure_bg':sure_bg,'unknown':unknown,'marked':orig}
+    # show_frames(frames)
+    # save_frames(frames,'binary_inv_dc/orig')
+    # cv.waitKey()
+
+
+if __name__ == '__main__':
+    print(args.file)
+    cap = cv.VideoCapture(args.file)
+    while not cap.isOpened():   
+        cap = cv.VideoCapture(args.file)
+        cv.waitKey(1000)
+        print ("Wait for the header")
+    while(True):
+        ret, frame = cap.read()
+        if ret:
+            frame = resize_frame(frame, 0.25)
+            dark = dark_channel(frame)[0]
+            # cv.imshow('OTSU+Dark',analyze(dark))
+            show_frames({'OTSU':analyze(frame),'dark':dark,'OTSU+DARK':analyze(dark),'orig':frame})
+            if cv.waitKey(1) & 0xFF == ord('q'):
+                break
+            if cv.waitKey(32) == ord(' '):
+                while(not cv.waitKey(32) == ord(' ')):
+                    continue
+    cv.waitKey(-1)

perception/__init__.py

@@ -3,12 +3,21 @@
 import perception.tasks.gate.GateSegmentationAlgoA as GateSegA
 import perception.tasks.gate.GateSegmentationAlgoB as GateSegB
 import perception.tasks.gate.GateSegmentationAlgoC as GateSegC
-# import perception.tasks as tasks
+import perception.tasks.segmentation.saliency_detection.MBD as MBD
+from perception.tasks.segmentation.COMB_SAL_BG import COMB_SAL_BG
+import perception.vis.TestTasks.BackgroundRemoval as BackgroundRemoval
+import perception.tasks.roulette.color_detection as RouletteColorDetector
+from perception.tasks.dice.DiceDetector import DiceDetector
 
 ALGOS = {
     'test': TestAlgo.TestAlgo,
     'gateseg': GateSeg.GateCenterAlgo,
     'gatesegA': GateSegA.GateSegmentationAlgoA,
     'gatesegB': GateSegB.GateSegmentationAlgoB,
-    'gatesegC': GateSegC.GateSegmentationAlgoC
+    'gatesegC': GateSegC.GateSegmentationAlgoC,
+    'MBD': MBD.MBD,
+    'bg-rm': BackgroundRemoval.BackgroundRemoval,
+    'combined': COMB_SAL_BG,
+    'roulette': RouletteColorDetector.RouletteColorDetector,
+    'dice': DiceDetector
 }

perception/misc/dark_channel/dehaze.py

@@ -0,0 +1,15 @@
+import cv2 as cv
+import numpy as np
+from .haze_removal import HazeRemoval
+from .utils import threshold_color_array
+from ...vis.TaskPerceiver import TaskPerceiver
+
+class Dehaze(TaskPerceiver):
+
+    def analyze(self, frame: np.ndarray, debug: bool, slider_vals: Dict[str, int]):
+        haze_removal_object = HazeRemoval(frame)
+        dark_channel = haze_removal_object.get_dark_channel(haze_removal_object.I)
+        A = haze_removal_object.get_atmosphere(dark_channel)
+        t = haze_removal_object.get_transmission(dark_channel, A)
+        recovered_image = haze_removal_object.get_recover_image(A, t)
+        return threshold_color_array(recovered_image)

perception/misc/dark_channel/depth_map.py

@@ -0,0 +1,17 @@
+import cv2 as cv
+import numpy as np
+from .haze_removal import HazeRemoval
+from .utils import threshold_color_array
+from ...vis.TaskPerceiver import TaskPerceiver
+
+class DepthMap(TaskPerceiver):
+
+    def __init__(self, beta=1):
+        self.beta = beta
+
+    def analyze(self, frame: np.ndarray, debug: bool, slider_vals: Dict[str, int]):
+        haze_removal_object = HazeRemoval(frame)
+        dark_channel = haze_removal_object.get_dark_channel(haze_removal_object.I)
+        A = haze_removal_object.get_atmosphere(dark_channel)
+        t = haze_removal_object.get_transmission(dark_channel, A)
+        return np.log(t) / -self.beta

perception/misc/dark_channel/handler.py

@@ -0,0 +1,42 @@
+import cv2 as cv
+import numpy as np
+from .haze_removal import HazeRemovel
+from .utils import threshold_color_array
+# from combinedFilter import init_combined_filter 
+
+# cap = cv.VideoCapture('/Users/karthikdharmarajan/Documents/URobotics/Course Footage/GOPR1146.MP4')
+# combined_filter = init_combined_filter()
+
+def rescale_frame(frame, percent=75):
+    width = int(frame.shape[1] * percent/ 100)
+    height = int(frame.shape[0] * percent/ 100)
+    dim = (width, height)
+    return cv.resize(frame, dim, interpolation =cv.INTER_AREA)
+
+def process_frame(frame):
+    haze_removal_object = HazeRemovel(frame)
+    dark_channel = haze_removal_object.get_dark_channel(haze_removal_object.I)
+    A = haze_removal_object.get_atmosphere(dark_channel)
+    t = haze_removal_object.get_transmission(dark_channel, A)
+    recover_image = haze_removal_object.get_recover_image(A, t)
+    return threshold_color_array(recover_image), t
+
+# while cap.isOpened():
+#     ret, img_in = cap.read()
+
+#     if ret:
+#         img_in = rescale_frame(img_in,30)
+#         recovered_img, depth_map = process_frame(img_in)
+#         thresholded_img_without_haze = cv.threshold(combined_filter(recovered_img), 0, 255, cv.THRESH_BINARY | cv.THRESH_OTSU)[1]
+#         threshold_img_haze = cv.threshold(combined_filter(img_in), 0, 255, cv.THRESH_BINARY | cv.THRESH_OTSU)[1]
+#         cv.imshow('img_in', img_in)
+#         cv.imshow('recovered_img', recovered_img)
+#         cv.imshow('thresholded_img_haze', threshold_img_haze)
+#         cv.imshow('threshold_img_without_haze', thresholded_img_without_haze)
+#         cv.imshow('depth_map', depth_map)
+
+#     if cv.waitKey(1) & 0xFF == ord('q'):
+#         break
+
+# cap.release()
+# cv.destroyAllWindows()

perception/misc/dark_channel/haze_removal.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+import os
+
+from PIL import Image
+import cv2
+import numpy as np
+
+from .utils import number_to_integral
+
+
+class HazeRemovel:
+
+    def __init__(self, image, refine=True, local_patch_size=15,
+                 omega=0.95, percentage=0.001, tmin=0.1):
+        self.refine = refine
+        self.local_patch_size = local_patch_size
+        self.omega = omega
+        self.percentage = percentage
+        self.tmin = tmin
+        self.image = image
+        self.I = self.image.astype(np.float64)
+        self.height, self.width, _ = self.I.shape
+
+    def get_dark_channel(self, image):
+        min_image = image.min(axis=2)
+        kernel = cv2.getStructuringElement(
+            cv2.MORPH_RECT,
+            (self.local_patch_size, self.local_patch_size)
+        )
+        dark_channel = cv2.erode(min_image, kernel).astype(np.uint8)
+        return dark_channel
+
+    def get_atmosphere(self, dark_channel):
+        img_size = self.height * self.width
+        flat_image = self.I.reshape(img_size, 3)
+        flat_dark = dark_channel.ravel()
+        pixel_count = number_to_integral(img_size * self.percentage)
+        search_idx = flat_dark.argsort()[-pixel_count:]
+        a = np.mean(flat_image.take(search_idx, axis=0), axis=0)
+        return a.astype(np.uint8)
+
+    def get_transmission(self, dark_channel, A):
+        transmission = 1 - self.omega * \
+            self.get_dark_channel(self.I / A * 255.0) / 255.0
+        if self.refine:
+            transmission = self.get_refined_transmission(transmission)
+        return transmission
+
+    def get_refined_transmission(self, transmission):
+        gray = self.image.min(axis=2)
+        t = (transmission * 255).astype(np.uint8)
+        refined_transmission = cv2.ximgproc.guidedFilter(gray, t, 40, 1e-2)
+        return refined_transmission / 255
+
+    def get_recover_image(self, A, transmission):
+        t = np.maximum(transmission, self.tmin)
+        tiled_t = np.zeros_like(self.I)
+        tiled_t[:, :, 0] = tiled_t[:, :, 1] = tiled_t[:, :, 2] = t
+        return (self.I - A) / tiled_t + A

perception/misc/dark_channel/utils.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+import numpy as np
+
+
+def number_to_integral(number):
+    return int(np.ceil(number))
+
+
+def threshold_color_array(src):
+    return np.maximum(np.minimum(src, 255), 0).astype(np.uint8)

perception/tasks/dice/DiceDetector.py

@@ -0,0 +1,47 @@
+import cv2 as cv
+import numpy as np
+from perception.tasks.TaskPerceiver import TaskPerceiver
+from typing import Dict
+from perception.tasks.segmentation.COMB_SAL_BG import COMB_SAL_BG
+import queue
+
+class DiceDetector(TaskPerceiver):
+    def __init__(self, **kwargs):
+        super().__init__(heuristic_threshold=((5,255), 35), run_both=((0,1),0), 
+            centroid_distance_weight=((0,200),1), area_percentage_weight=((0,200),60), num_contours=((1,5), 1))
+        self.sal = COMB_SAL_BG()
+        self.sal.set_num_contours(1)
+        self.sal.use_saliency = False
+        self.interpolator = RectangleInterpolator(20)
+
+    def analyze(self, frame: np.ndarray, debug: bool, slider_vals: Dict[str, int]):
+        contours = self.sal.analyze(frame, False, slider_vals)
+        if len(contours) > 0:
+            for contour in contours:
+                x,y,w,h = cv.boundingRect(contour)
+                cv.rectangle(frame,(x,y),(x+w,y+h),(255,0,0),2)
+                self.interpolator.insert_new_rectangle(x, y, w, h)
+        avg_coord1, avg_coord2 = self.interpolator.get_avg_coordinates()
+        cv.rectangle(frame,avg_coord1,avg_coord2,(255,0,255),2)
+        return contours, [frame]
+
+class RectangleInterpolator():
+
+    def __init__(self, num_rectangles):
+        self.contour_queue = queue.Queue(maxsize=num_rectangles)
+        self.coord1Sum = (0, 0)
+        self.coord2Sum = (0, 0)
+
+    def insert_new_rectangle(self, x, y, w, h):
+        if self.contour_queue.full():
+            value = self.contour_queue.get()
+            self.coord1Sum = (self.coord1Sum[0] - value[0][0], self.coord1Sum[1] - value[0][1])
+            self.coord2Sum = (self.coord2Sum[0] - value[1][0], self.coord2Sum[1] - value[1][1])
+        self.contour_queue.put([(x, y), (x+w, y+h)])
+        self.coord1Sum = (self.coord1Sum[0] + x, self.coord1Sum[1] + y)
+        self.coord2Sum = (self.coord2Sum[0] + x + w, self.coord2Sum[1] + y + h)
+
+    def get_avg_coordinates(self):
+        avg_coord1 = (int(self.coord1Sum[0] / self.contour_queue.qsize()), int(self.coord1Sum[1] / self.contour_queue.qsize()))
+        avg_coord2 = (int(self.coord2Sum[0] / self.contour_queue.qsize()), int(self.coord2Sum[1] / self.contour_queue.qsize()))
+        return avg_coord1, avg_coord2