diff --git a/tfjs/smartWebCamera/README.md b/tfjs/smartWebCamera/README.md new file mode 100644 index 00000000000..75eae1c32ee --- /dev/null +++ b/tfjs/smartWebCamera/README.md @@ -0,0 +1,9 @@ +# Make a smart webcam in JavaScript with a TensorFlow.js pre-trained Machine Learning model + +#### Reference Codelab: [Codelab](https://codelabs.developers.google.com/codelabs/tensorflowjs-object-detection?hl=en&continue=https%3A%2F%2Fcodelabs.developers.google.com%2F#0) + +### What we build + +- Create a webpage that uses machine learning directly in the web browser via TensorFlow.js to classify and detect common objects, (yes, including more than one at a time), from a live webcam stream. +- Supercharge your regular webcam to identify objects and get the coordinates of the bounding box for each object it finds +- Highlight the found object in the video stream diff --git a/tfjs/smartWebCamera/index.html b/tfjs/smartWebCamera/index.html new file mode 100644 index 00000000000..83ed032bb67 --- /dev/null +++ b/tfjs/smartWebCamera/index.html @@ -0,0 +1,58 @@ + + + + + + Multiple object detection using pre trained model in TensorFlow.js + + + + + + + +

Multiple object detection using pre trained model in TensorFlow.js

+ +

Wait for the model to load before clicking the button to enable the webcam + - at which point it will become visible to use.

+ + + + + + + + + + + + + + + + + + diff --git a/tfjs/smartWebCamera/script.js b/tfjs/smartWebCamera/script.js new file mode 100644 index 00000000000..321304fc6c0 --- /dev/null +++ b/tfjs/smartWebCamera/script.js @@ -0,0 +1,121 @@ +/* + - Referencing key DOM elements + - Check for webcam support + - Fetching the webcam stream + - Loading the model + +*/ +// const status = document.getElementById('status'); +// status.innerText = 'Loaded TensorFlow.js - version: ' + tf.version.tfjs; + +const video = document.getElementById('webcam'); +const liveView = document.getElementById('liveView'); +const demosSection = document.getElementById('demos'); +const enableWebcamButton = document.getElementById('webcamButton'); + +// Check if webcam access is supported. +function getUserMediaSupported() { + return !!(navigator.mediaDevices && + navigator.mediaDevices.getUserMedia); + } + +// If webcam supported, add event listener to button for when user +// wants to activate it to call enableCam function which we will +// define in the next step. +if (getUserMediaSupported()) { + enableWebcamButton.addEventListener('click', enableCam); + // console.log("till loading camera done") + } else { + console.warn('getUserMedia() is not supported by your browser'); + } + + // Placeholder function for next step. Paste over this in the next step. +// function enableCam(event) { +// } + +// Enable the live webcam view and start classification. +function enableCam(event) { + // Only continue if the COCO-SSD has finished loading. + if (!model) { + return; + } + + // Hide the button once clicked. + event.target.classList.add('removed'); + + // getUsermedia parameters to force video but not audio. + const constraints = { + video: true + }; + + // Activate the webcam stream. + navigator.mediaDevices.getUserMedia(constraints).then(function(stream) { + video.srcObject = stream; + video.addEventListener('loadeddata', predictWebcam); + }); + } + +// // Placeholder function for next step. +// function predictWebcam() { +// } + +// // Pretend model has loaded so we can try out the webcam code. +// var model = true; +// demosSection.classList.remove('invisible'); + +// Store the resulting model in the global scope of our app. +var model = undefined; + +// Before we can use COCO-SSD class we must wait for it to finish +// loading. Machine Learning models can be large and take a moment +// to get everything needed to run. +// Note: cocoSsd is an external object loaded from our index.html +// script tag import so ignore any warning in Glitch. +cocoSsd.load().then(function (loadedModel) { + model = loadedModel; + // Show demo section now model is ready to use. + demosSection.classList.remove('invisible'); +}); + +var children = []; + +function predictWebcam() { + // Now let's start classifying a frame in the stream. + model.detect(video).then(function (predictions) { + // Remove any highlighting we did previous frame. + for (let i = 0; i < children.length; i++) { + liveView.removeChild(children[i]); + } + children.splice(0); + + // Now lets loop through predictions and draw them to the live view if + // they have a high confidence score. + for (let n = 0; n < predictions.length; n++) { + // If we are over 66% sure we are sure we classified it right, draw it! + if (predictions[n].score > 0.66) { + const p = document.createElement('p'); + p.innerText = predictions[n].class + ' - with ' + + Math.round(parseFloat(predictions[n].score) * 100) + + '% confidence.'; + p.style = 'margin-left: ' + predictions[n].bbox[0] + 'px; margin-top: ' + + (predictions[n].bbox[1] - 10) + 'px; width: ' + + (predictions[n].bbox[2] - 10) + 'px; top: 0; left: 0;'; + + const highlighter = document.createElement('div'); + highlighter.setAttribute('class', 'highlighter'); + highlighter.style = 'left: ' + predictions[n].bbox[0] + 'px; top: ' + + predictions[n].bbox[1] + 'px; width: ' + + predictions[n].bbox[2] + 'px; height: ' + + predictions[n].bbox[3] + 'px;'; + + liveView.appendChild(highlighter); + liveView.appendChild(p); + children.push(highlighter); + children.push(p); + } + } + + // Call this function again to keep predicting when the browser is ready. + window.requestAnimationFrame(predictWebcam); + }); +} diff --git a/tfjs/smartWebCamera/style.css b/tfjs/smartWebCamera/style.css new file mode 100644 index 00000000000..3d108430eee --- /dev/null +++ b/tfjs/smartWebCamera/style.css @@ -0,0 +1,52 @@ +body { + font-family: helvetica, arial, sans-serif; + margin: 2em; + color: #3D3D3D; +} + +h1 { + font-style: italic; + color: #FF6F00; +} + +video { + display: block; +} + +section { + opacity: 1; + transition: opacity 500ms ease-in-out; +} + +.removed { + display: none; +} + +.invisible { + opacity: 0.2; +} + +.camView { + position: relative; + float: left; + width: calc(100% - 20px); + margin: 10px; + cursor: pointer; +} + +.camView p { + position: absolute; + padding: 5px; + background-color: rgba(255, 111, 0, 0.85); + color: #FFF; + border: 1px dashed rgba(255, 255, 255, 0.7); + z-index: 2; + font-size: 12px; +} + +.highlighter { + background: rgba(0, 255, 0, 0.25); + border: 1px dashed #fff; + z-index: 1; + position: absolute; +}