diff --git a/FacialFeatureDetection/FacialFeaturesDetection.ipynb b/FacialFeatureDetection/FacialFeaturesDetection.ipynb new file mode 100644 index 00000000..fd5367ba --- /dev/null +++ b/FacialFeatureDetection/FacialFeaturesDetection.ipynb @@ -0,0 +1,362 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "FacialFeaturesDetection.ipynb", + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "code", + "metadata": { + "id": "_0uObd7elchO" + }, + "source": [ + "import numpy as np \n", + "import cv2 \n", + "import dlib \n", + "from scipy.spatial import distance as dist \n", + "from scipy.spatial import ConvexHull\n", + "\n", + "def eye_size(eye):\n", + " eyeWidth = dist.euclidean(eye[0], eye[3])\n", + " hull = ConvexHull(eye)\n", + " eyeCenter = np.mean(eye[hull.vertices, :], axis=0)\n", + " eyeCenter = eyeCenter.astype(int)\n", + "\n", + " return int(eyeWidth), eyeCenter\n", + "\n", + "def place_eye(frame, eyeCenter, eyeSize, filter_img, mask, mask_inv): \n", + "\teyeSize = int(eyeSize * 1.5) \n", + "\n", + "\tx1 = int(eyeCenter[0,0] - (eyeSize/2)) \n", + "\tx2 = int(eyeCenter[0,0] + (eyeSize/2)) \n", + "\ty1 = int(eyeCenter[0,1] - (eyeSize/2)) \n", + "\ty2 = int(eyeCenter[0,1] + (eyeSize/2)) \n", + "\n", + "\th, w = frame.shape[:2] \n", + "\n", + "\t# check for clipping \n", + "\tif x1 < 0: \n", + "\t x1 = 0 \n", + "\tif y1 < 0: \n", + "\t y1 = 0 \n", + "\tif x2 > w: \n", + "\t x2 = w \n", + "\tif y2 > h: \n", + "\t y2 = h \n", + "\n", + "\t# re-calculate the size to avoid clipping \n", + "\teyeOverlayWidth = x2 - x1 \n", + "\teyeOverlayHeight = y2 - y1 \n", + "\n", + "\t# calculate the masks for the overlay \n", + "\teyeOverlay = cv2.resize(filter_img, (eyeOverlayWidth,eyeOverlayHeight), interpolation = cv2.INTER_AREA) \n", + "\tmask = cv2.resize(mask, (eyeOverlayWidth,eyeOverlayHeight), interpolation = cv2.INTER_AREA) \n", + "\tmask_inv = cv2.resize(mask_inv, (eyeOverlayWidth,eyeOverlayHeight), interpolation = cv2.INTER_AREA) \n", + "\n", + "\t# take ROI for the verlay from background, equal to size of the overlay image \n", + "\troi = frame[y1:y2, x1:x2] \n", + "\n", + "\t# roi_bg contains the original image only where the overlay is not, in the region that is the size of the overlay. \n", + "\troi_bg = cv2.bitwise_and(roi,roi,mask = mask_inv) \n", + "\n", + "\t# roi_fg contains the image pixels of the overlay only where the overlay should be \n", + "\troi_fg = cv2.bitwise_and(eyeOverlay,eyeOverlay,mask = mask) \n", + "\n", + "\t# join the roi_bg and roi_fg \n", + "\tdst = cv2.add(roi_bg,roi_fg) \n", + "\n", + "\t# place the joined image, saved to dst back over the original image \n", + "\tframe[y1:y2, x1:x2] = dst\n", + "\n", + "\n", + "def funny_eyes(landmark, LEFT_EYE_POINTS, RIGHT_EYE_POINTS, frame, filter_img, mask, mask_inv):\n", + " left_eye = landmark[LEFT_EYE_POINTS]\n", + " right_eye = landmark[RIGHT_EYE_POINTS]\n", + " lefteyesize, lefteyecenter = eye_size(left_eye)\n", + " righteyesize, righteyecenter = eye_size(right_eye)\n", + " place_eye(frame, lefteyecenter, lefteyesize, filter_img, mask, mask_inv) \n", + " place_eye(frame, righteyecenter, righteyesize, filter_img, mask, mask_inv)\n", + "\n", + "\n", + "def nose_dimensions(nose):\n", + " height = dist.euclidean(nose[0], nose[3])\n", + " width = dist.euclidean(nose[4], nose[8])\n", + " return int(height), int(width)\n", + "\n", + "\n", + "def place_moustache(frame, nose, nose_w, nose_h, filter_img, mask, mask_inv):\n", + " orig_height, orig_width = filter_img.shape[:2]\n", + " mustacheWidth = int(3 * nose_w)\n", + " mustacheHeight = int(mustacheWidth * orig_height / orig_width)\n", + "\n", + " x1 = int(nose[8, 0] - nose_w - (mustacheWidth/4))\n", + " x2 = int(nose[8, 0] + (mustacheWidth/4))\n", + " y1 = int(nose[8, 1] - (mustacheHeight/2))\n", + " y2 = int(nose[8, 1] + (mustacheHeight/2))\n", + "\n", + " h, w = frame.shape[:2]\n", + "\n", + " if x1 < 0:\n", + " x1 = 0\n", + " if y1 < 0:\n", + " y1 = 0\n", + " if x2 > w:\n", + " x2 = w\n", + " if y2 > h:\n", + " y2 = h\n", + "\n", + " mustacheWidth = int(x2 - x1)\n", + " mustacheHeight = int(y2 - y1)\n", + "\n", + " # Re-size the original image and the masks to the mustache sizes\n", + " # calcualted above\n", + " mustache = cv2.resize(filter_img, (mustacheWidth,mustacheHeight), interpolation = cv2.INTER_AREA)\n", + " mask = cv2.resize(mask, (mustacheWidth,mustacheHeight), interpolation = cv2.INTER_AREA)\n", + " mask_inv = cv2.resize(mask_inv, (mustacheWidth,mustacheHeight), interpolation = cv2.INTER_AREA)\n", + "\n", + " # take ROI for mustache from background equal to size of mustache image\n", + " roi = frame[y1:y2, x1:x2]\n", + "\n", + " # roi_bg contains the original image only where the mustache is not\n", + " # in the region that is the size of the mustache.\n", + " roi_bg = cv2.bitwise_and(roi,roi,mask = mask_inv)\n", + "\n", + " # roi_fg contains the image of the mustache only where the mustache is\n", + " roi_fg = cv2.bitwise_and(mustache,mustache,mask = mask)\n", + "\n", + " # join the roi_bg and roi_fg\n", + " dst = cv2.add(roi_bg,roi_fg)\n", + "\n", + " # place the joined image, saved to dst back over the original image\n", + " frame[y1:y2, x1:x2] = dst\n", + "\n", + "def santa_moustache(frame, nose, nose_w, nose_h, filter_img, mask, mask_inv):\n", + " orig_height, orig_width = filter_img.shape[:2]\n", + " mustacheWidth = int(5 * nose_w)\n", + " mustacheHeight = int(mustacheWidth * orig_height / orig_width)\n", + "\n", + " x1 = int(nose[8, 0] - nose_w - (mustacheWidth/4))\n", + " x2 = int(nose[8, 0] + (mustacheWidth/4))\n", + " y1 = int(nose[8, 1])\n", + " y2 = int(nose[8, 1] + (mustacheHeight/2))\n", + "\n", + " h, w = frame.shape[:2]\n", + "\n", + " if x1 < 0:\n", + " x1 = 0\n", + " if y1 < 0:\n", + " y1 = 0\n", + " if x2 > w:\n", + " x2 = w\n", + " if y2 > h:\n", + " y2 = h\n", + "\n", + " mustacheWidth = int(x2 - x1)\n", + " mustacheHeight = int(y2 - y1)\n", + "\n", + " # Re-size the original image and the masks to the mustache sizes\n", + " # calcualted above\n", + " mustache = cv2.resize(filter_img, (mustacheWidth,mustacheHeight), interpolation = cv2.INTER_AREA)\n", + " mask = cv2.resize(mask, (mustacheWidth,mustacheHeight), interpolation = cv2.INTER_AREA)\n", + " mask_inv = cv2.resize(mask_inv, (mustacheWidth,mustacheHeight), interpolation = cv2.INTER_AREA)\n", + "\n", + " # take ROI for mustache from background equal to size of mustache image\n", + " roi = frame[y1:y2, x1:x2]\n", + "\n", + " # roi_bg contains the original image only where the mustache is not\n", + " # in the region that is the size of the mustache.\n", + " roi_bg = cv2.bitwise_and(roi, roi, mask = mask_inv)\n", + "\n", + " # roi_fg contains the image of the mustache only where the mustache is\n", + " roi_fg = cv2.bitwise_and(mustache, mustache, mask = mask)\n", + "\n", + " # join the roi_bg and roi_fg\n", + " dst = cv2.add(roi_bg,roi_fg)\n", + "\n", + " # place the joined image, saved to dst back over the original image\n", + " frame[y1:y2, x1:x2] = dst\n", + "\n", + "def santa_hat(frame, face, face_w, face_h, filter_img, mask, mask_inv):\n", + "\toriginal_hat_h, original_hat_w = filter_img.shape[:2]\n", + "\that_width = int(1.2 * face_w)\n", + "\that_height = int(hat_width * original_hat_h / original_hat_w)\n", + "\t# print(hat_height)\n", + "\tface_x2, face_y1 = face\n", + "\t#setting location of coordinates of witch\n", + "\that_x1 = face_x2 - int(face_w/2) - int(hat_width/2) - 2\n", + "\that_x2 = hat_x1 + hat_width\n", + "\that_y1 = face_y1\n", + "\that_y2 = int(hat_y1 - (hat_height*0.9))\n", + "\n", + "\th, w = frame.shape[:2]\n", + "\n", + "\t#Conditions to check if any out of frame\n", + "\tif hat_x1 < 0:\n", + "\t hat_x1 = 0\n", + "\tif hat_y2 < 0:\n", + "\t hat_y2 = 0\n", + "\tif hat_x2 > w:\n", + "\t hat_x2 = w\n", + "\tif hat_y1 > h:\n", + "\t hat_y1 = h\n", + "\n", + "\n", + "\that_width = int(hat_x2 - hat_x1)\n", + "\that_height = int(hat_y1 - hat_y2)\n", + "\n", + "\t#resizing witch hat image to fit on face\n", + "\twitch = cv2.resize(filter_img, (hat_width,hat_height), interpolation = cv2.INTER_AREA)\n", + "\tmask = cv2.resize(mask, (hat_width,hat_height), interpolation = cv2.INTER_AREA)\n", + "\tmask_inv = cv2.resize(mask_inv, (hat_width,hat_height), interpolation = cv2.INTER_AREA)\n", + "\n", + "\t#take ROI for witch from background that is equal to size of witch image\n", + "\troi = frame[hat_y2:hat_y1, hat_x1:hat_x2]\n", + "\n", + "\t#original image in background (bg) where witch is not\n", + "\troi_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n", + "\troi_fg = cv2.bitwise_and(witch, witch, mask=mask)\n", + "\tdst = cv2.add(roi_bg,roi_fg)\n", + "\n", + "\t#put back in original image\n", + "\tframe[hat_y2:hat_y1, hat_x1:hat_x2] = dst\n", + "\n", + "def santaclaus(landmarks, NOSE_POINTS, face, frame, filter_img_1, filter_img_2, mask_1, mask_2, mask_inv_1, mask_inv_2):\n", + "\tnose = landmarks[NOSE_POINTS]\n", + "\tnose_h, nose_w = nose_dimensions(nose)\n", + "\n", + "\tx1, x2, y1, y2 = face\n", + "\tface_h = int(y1 - y2)\n", + "\tface_w = int(x2 - x1)\n", + "\tFACE = [x2, y1]\n", + "\t# frame = cv2.circle(frame, (x2,y1), radius=1, color=(0, 0, 255))\n", + "\t# cv2.rectangle(frame, (nose[8, 0]-nose_w, nose[8, 1]-nose_h), (nose[8, 0], nose[8, 1]), (255, 0, 0), 2)\n", + "\n", + "\tsanta_moustache(frame, nose, nose_w, nose_h, filter_img_1, mask_1, mask_inv_1)\n", + "\tsanta_hat(frame, FACE, face_w, face_h, filter_img_2, mask_2, mask_inv_2)\n", + "\n", + "def mustache(landmark, NOSE_POINTS, frame, filter_img, mask, mask_inv):\n", + " nose = landmark[NOSE_POINTS]\n", + " nose_h, nose_w = nose_dimensions(nose)\n", + " # cv2.rectangle(frame, (nose[8, 0]-nose_w, nose[8, 1]-nose_h), (nose[8, 0], nose[8, 1]), (255, 0, 0), 2)\n", + "\n", + " place_moustache(frame, nose, nose_w, nose_h, filter_img, mask, mask_inv)\n", + "\n", + "def create_mask(filter_img):\n", + "\tmask = filter_img[:, :, 3]\n", + "\tmask_inv = cv2.bitwise_not(mask)\n", + "\tfilter_img = filter_img[:, :, 0:3]\n", + "\treturn mask, mask_inv, filter_img\n", + "\n", + "def main(filter=None):\n", + " print('IMPORTANT!!!!!!')\n", + " print('To close the frame, press \"q\" key on your keyboard!')\n", + " detector = dlib.get_frontal_face_detector()\n", + " predictor = dlib.shape_predictor(\"model/shape_predictor_68_face_landmarks.dat\")\n", + "\n", + " FULL_POINTS = list(range(0, 68)) \n", + " FACE_POINTS = list(range(17, 68)) \n", + " \n", + " JAWLINE_POINTS = list(range(0, 17)) \n", + " RIGHT_EYEBROW_POINTS = list(range(17, 22)) \n", + " LEFT_EYEBROW_POINTS = list(range(22, 27)) \n", + " NOSE_POINTS = list(range(27, 36)) \n", + " RIGHT_EYE_POINTS = list(range(36, 42)) \n", + " LEFT_EYE_POINTS = list(range(42, 48)) \n", + " MOUTH_OUTLINE_POINTS = list(range(48, 61)) \n", + " MOUTH_INNER_POINTS = list(range(61, 68))\n", + "\n", + " if filter == 'funnyeye':\n", + " filter_img = cv2.imread('filters/eye.png', -1)\n", + " mask, mask_inv, filter_img = create_mask(filter_img)\n", + " if filter == 'moustache':\n", + " filter_img = cv2.imread('filters/moustache.png', -1)\n", + " mask, mask_inv, filter_img = create_mask(filter_img)\n", + " if filter == 'santaclaus':\n", + " filter_img_1 = cv2.imread('filters/beard.png', -1)\n", + " filter_img_2 = cv2.imread('filters/santa-cap.png', -1)\n", + " mask_1, mask_inv_1, filter_img_1 = create_mask(filter_img_1)\n", + " mask_2, mask_inv_2, filter_img_2 = create_mask(filter_img_2)\n", + "\n", + " # filter_height, filter_width = filter_img.shape[:2]\n", + " cap = cv2.VideoCapture(0)\n", + "\n", + " while True:\n", + " _, frame = cap.read()\n", + " gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n", + "\n", + " faces = detector(gray)\n", + " for face in faces:\n", + " x1 = face.left()\n", + " y1 = face.top()\n", + " x2 = face.right()\n", + " y2 = face.bottom()\n", + " # cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 255, 0), 3)\n", + "\n", + " landmarks = predictor(gray, face)\n", + " face_landmark = []\n", + " for p in landmarks.parts():\n", + " face_landmark.append([p.x, p.y])\n", + "\n", + " landmark = np.matrix(face_landmark)\n", + " if filter == 'funnyeye':\n", + " funny_eyes(landmark, LEFT_EYE_POINTS, RIGHT_EYE_POINTS, frame, filter_img, mask, mask_inv)\n", + " if filter == 'moustache':\n", + " mustache(landmark, NOSE_POINTS, frame, filter_img, mask, mask_inv)\n", + " if filter == 'santaclaus':\n", + " santaclaus(landmark, NOSE_POINTS, [x1, x2, y1, y2], frame, filter_img_1, filter_img_2, mask_1, mask_2, mask_inv_1, mask_inv_2) \n", + "\n", + " cv2.imshow(\"Frame\", frame)\n", + "\n", + " key = cv2.waitKey(1)\n", + " if key == ord('q'):\n", + " break" + ], + "execution_count": 1, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "9NCZTye71QQR" + }, + "source": [ + "main(\"funnyeye\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "eDGFFm0H1SMu" + }, + "source": [ + "main(\"moustache\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "yrl2z0SH1TrT" + }, + "source": [ + "main(\"santaclaus\")" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/FacialFeatureDetection/README.md b/FacialFeatureDetection/README.md new file mode 100644 index 00000000..9201f186 --- /dev/null +++ b/FacialFeatureDetection/README.md @@ -0,0 +1,18 @@ +# Facial Feature Detection + +## About Project +In this project, I am making use dlib library and OpenCV to mimic the facial filters like Snapchat or Instagram. In today's time, facial filters have garnered immense popularity amongst the millennials and Gen-Z alike. + +Currently, the project supports basic filter props such as moustache, funny eyes and santa claus. But the possibilities of different facial filters are endless. + +## Implementation + +I have used pretrained weights to predict the facial landmarks of the person(s) on the live webcam. Dlib library is used to load these weights and make the prediction. + +Please refer to the picture below for the different facial landmarks that can be predicted using ```shape_predictor_68_face_landmarks.dat``` file. + +![img](https://camo.githubusercontent.com/04da00a85d1bb8f8aaeca0da27cccad8d63674896f8b858d60759bb3af98d25e/68747470733a2f2f63646e2d696d616765732d312e6d656469756d2e636f6d2f6d61782f313630302f312a393655542d44387553586a6c6e79767339445a546f672e706e67) + +Once the landmarks are identified, I am using OpenCV to load the filter props, resize them based on the size of face and then, superimpose them over the live-streaming frame for filter like effect. + +Filter to be applied is selected using function argument. \ No newline at end of file diff --git a/FacialFeatureDetection/filters/beard.png b/FacialFeatureDetection/filters/beard.png new file mode 100644 index 00000000..41fe1fa8 Binary files /dev/null and b/FacialFeatureDetection/filters/beard.png differ diff --git a/FacialFeatureDetection/filters/eye.png b/FacialFeatureDetection/filters/eye.png new file mode 100644 index 00000000..6b3ef074 Binary files /dev/null and b/FacialFeatureDetection/filters/eye.png differ diff --git a/FacialFeatureDetection/filters/moustache.png b/FacialFeatureDetection/filters/moustache.png new file mode 100644 index 00000000..081f0e34 Binary files /dev/null and b/FacialFeatureDetection/filters/moustache.png differ diff --git a/FacialFeatureDetection/filters/moustache_2.png b/FacialFeatureDetection/filters/moustache_2.png new file mode 100644 index 00000000..062c34cb Binary files /dev/null and b/FacialFeatureDetection/filters/moustache_2.png differ diff --git a/FacialFeatureDetection/filters/mustache.png b/FacialFeatureDetection/filters/mustache.png new file mode 100644 index 00000000..db924832 Binary files /dev/null and b/FacialFeatureDetection/filters/mustache.png differ diff --git a/FacialFeatureDetection/filters/santa-cap.png b/FacialFeatureDetection/filters/santa-cap.png new file mode 100644 index 00000000..75c3035d Binary files /dev/null and b/FacialFeatureDetection/filters/santa-cap.png differ diff --git a/FacialFeatureDetection/filters/sunglasses_1.png b/FacialFeatureDetection/filters/sunglasses_1.png new file mode 100644 index 00000000..d0a492fc Binary files /dev/null and b/FacialFeatureDetection/filters/sunglasses_1.png differ diff --git a/FacialFeatureDetection/filters/sunglasses_2.png b/FacialFeatureDetection/filters/sunglasses_2.png new file mode 100644 index 00000000..47d8d508 Binary files /dev/null and b/FacialFeatureDetection/filters/sunglasses_2.png differ diff --git a/FacialFeatureDetection/model/shape_predictor_68_face_landmarks.dat b/FacialFeatureDetection/model/shape_predictor_68_face_landmarks.dat new file mode 100644 index 00000000..e0ec20d6 Binary files /dev/null and b/FacialFeatureDetection/model/shape_predictor_68_face_landmarks.dat differ diff --git a/FacialFeatureDetection/requirements.txt b/FacialFeatureDetection/requirements.txt new file mode 100644 index 00000000..e9582538 --- /dev/null +++ b/FacialFeatureDetection/requirements.txt @@ -0,0 +1,4 @@ +opencv-python==4.1.1.26 +numpy==1.19.2 +dlib==19.18.0 +scipy==1.4.1 \ No newline at end of file