Object Detection using node-red-contrib-tfjs-coco-ssd

General
48

No python script required anymore, everything is in js. OpenCV is in this simpler case just drawing all the boxes on top of the image. With OpenCV available in NR, much more can be done in terms of image manipulation

Requires the installation of opencv4nodejs (in a Pi3 I did like this (from /home/pi folder)):

  • Increase the CONF_SWAPSIZE from 100 to 2048:
    sudo nano /etc/dphys-swapfile
    Reboot
  • Run the installation:
    npm install --force --save opencv4nodejs
  • Set the CONF_SWAPSIZE back to 100 :
    sudo nano /etc/dphys-swapfile
    Reboot

In my settings.js I have added:
cv2:require('opencv4nodejs'),

This is the code in the function node:

var image = context.get('mat')||undefined;
var cv = global.get('cv2');
var detections = [];

const rows = 240; // height
const cols = 320; // width

if(typeof(msg.payload)==="string"){
    const buffer = Buffer.from(msg.payload,'base64');
    image = cv.imdecode(buffer); //Image is now represented as Mat
    context.set('mat', image);
}else{
    detections = msg.payload;
    showBoxes();
    //cv.imwrite('/home/pi/Pictures/img.jpg', image);
    // convert Mat to base64 encoded jpg image
    const outBase64 =  cv.imencode('.jpg', image).toString('base64'); // Perform base64 encoding
    msg.payload = outBase64;
    node.send(msg);
}

function writeBox(element) {
    let box = element;
    let x = parseInt(box.bbox[0]);
    let y = parseInt(box.bbox[1]);
    let width = parseInt(box.bbox[2]);
    let height= parseInt(box.bbox[3]);
    let theClass = box["class"];
    let theScore = box["score"].toFixed(2)*100;
    //node.warn(theScore);
    const blue = new cv.Vec3(255, 0, 0);
    const font = cv.FONT_HERSHEY_SIMPLEX;
    const green = new cv.Vec3(0, 255, 0);
    image.drawRectangle(new cv.Point2(x, y),new cv.Point2(x+width, y+height),blue,2);
    image.putText(theClass+' '+theScore+'%', new cv.Point2(x, y-5), font, 0.5, green, 2);
}

function showBoxes() {
    detections.forEach(element => writeBox(element))
}

This is the flow:

[{"id":"379c521a.e9d72e","type":"comment","z":"2d489fd9.1eedd","name":"NR Object detection","info":"","x":170,"y":40,"wires":[]},{"id":"8a69170a.2a6d78","type":"ui_template","z":"2d489fd9.1eedd","group":"31b5112.2d60fee","name":"highlightPict","order":2,"width":"24","height":"14","format":"<!DOCTYPE html>\n<html>\n<style type=\"text/css\"> \n.wrapper {\n    position:absolute;\n    width:320px;\n    height:240px;\n    border:2px solid grey;\n}\n</style> \n\n<script type=\"text/javascript\">\nscope.$watch('msg', function(msg) {\n    inMessage(msg);\n});\n\nfunction inMessage(event) {\n    if (event.payload) {\n        document.getElementById('cX').src = \"data:image/jpg;base64,\"+event.payload;\n    }\n}\n\n</script>\n<body>\n<div class=\"wrapper\">\n     <img src=\"\" id=\"cX\" />\n</div>\n</body>\n</html>\n\n\n\n","storeOutMessages":true,"fwdInMessages":true,"templateScope":"local","x":1070,"y":80,"wires":[[]]},{"id":"2fccc750.eba658","type":"tensorflowCoco","z":"2d489fd9.1eedd","name":"","model":"","scoreThreshold":"","x":610,"y":260,"wires":[["f37f4cb9.3def6","9a7b8548.3747c8","f31c5417.7fee68"]]},{"id":"2085ac81.9563d4","type":"base64","z":"2d489fd9.1eedd","name":"","action":"str","property":"payload","x":610,"y":80,"wires":[["b681c523.3bd768"]]},{"id":"f37f4cb9.3def6","type":"ui_table","z":"2d489fd9.1eedd","group":"31b5112.2d60fee","name":"table detection","order":8,"width":10,"height":"12","columns":[],"outputs":0,"cts":false,"x":830,"y":260,"wires":[]},{"id":"2ac71df3.5060f2","type":"http request","z":"2d489fd9.1eedd","name":"","method":"GET","ret":"bin","paytoqs":false,"url":"https://loremflickr.com/320/240/sport","tls":"","persist":false,"proxy":"","authType":"","x":360,"y":170,"wires":[["2fccc750.eba658","2085ac81.9563d4","9a7b8548.3747c8"]]},{"id":"dfc95a0d.0305c8","type":"ui_button","z":"2d489fd9.1eedd","name":"","group":"31b5112.2d60fee","order":6,"width":1,"height":1,"passthru":true,"label":"New Picture","tooltip":"","color":"","bgcolor":"","icon":"","payload":"","payloadType":"str","topic":"","x":150,"y":170,"wires":[["2ac71df3.5060f2"]]},{"id":"fba73baf.e43ef8","type":"inject","z":"2d489fd9.1eedd","name":"test","topic":"","payload":"","payloadType":"date","repeat":"","crontab":"","once":false,"onceDelay":0.1,"x":130,"y":80,"wires":[["dfc95a0d.0305c8"]]},{"id":"9a7b8548.3747c8","type":"function","z":"2d489fd9.1eedd","name":"","func":"oldT = context.get('prev')||0;\nvar time = new Date().getTime();\nif(oldT === 0){\n    context.set('prev', time);\n}\nif(oldT>0){\n    node.warn(time-oldT);\n    context.set('prev', 0);\n}\n","outputs":0,"noerr":0,"x":610,"y":170,"wires":[]},{"id":"3bf60ae8.caf4f6","type":"mqtt in","z":"2d489fd9.1eedd","name":"","topic":"image/#","qos":"2","datatype":"auto","broker":"2a019090.5ba4d","x":140,"y":260,"wires":[["adb4835c.d0472"]]},{"id":"adb4835c.d0472","type":"switch","z":"2d489fd9.1eedd","name":"","property":"payload","propertyType":"msg","rules":[{"t":"istype","v":"buffer","vt":"buffer"}],"checkall":"true","repair":false,"outputs":1,"x":360,"y":260,"wires":[["2085ac81.9563d4","9a7b8548.3747c8","2fccc750.eba658"]]},{"id":"f31c5417.7fee68","type":"switch","z":"2d489fd9.1eedd","name":"","property":"classes","propertyType":"msg","rules":[{"t":"hask","v":"person","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":830,"y":170,"wires":[["b681c523.3bd768"]]},{"id":"b681c523.3bd768","type":"function","z":"2d489fd9.1eedd","name":"","func":"var image = context.get('mat')||undefined;\nvar cv = global.get('cv2');\nvar detections = [];\n\nconst rows = 240; // height\nconst cols = 320; // width\n\nif(typeof(msg.payload)===\"string\"){\n    const buffer = Buffer.from(msg.payload,'base64');\n    image = cv.imdecode(buffer); //Image is now represented as Mat\n    context.set('mat', image);\n}else{\n    detections = msg.payload;\n    showBoxes();\n    //cv.imwrite('/home/pi/Pictures/img.jpg', image);\n    // convert Mat to base64 encoded jpg image\n    const outBase64 =  cv.imencode('.jpg', image).toString('base64'); // Perform base64 encoding\n    msg.payload = outBase64;\n    node.send(msg);\n}\n\nfunction writeBox(element) {\n    let box = element;\n    let x = parseInt(box.bbox[0]);\n    let y = parseInt(box.bbox[1]);\n    let width = parseInt(box.bbox[2]);\n    let height= parseInt(box.bbox[3]);\n    let theClass = box[\"class\"];\n    let theScore = box[\"score\"].toFixed(2)*100;\n    //node.warn(theScore);\n    const blue = new cv.Vec3(255, 0, 0);\n    const font = cv.FONT_HERSHEY_SIMPLEX;\n    const green = new cv.Vec3(0, 255, 0);\n    image.drawRectangle(new cv.Point2(x, y),new cv.Point2(x+width, y+height),blue,2);\n    image.putText(theClass+' '+theScore+'%', new cv.Point2(x, y-5), font, 0.5, green, 2);\n}\n\nfunction showBoxes() {\n    detections.forEach(element => writeBox(element))\n}\n\n","outputs":1,"noerr":0,"x":830,"y":80,"wires":[["d4e78cd3.ba85c","8a69170a.2a6d78","831edb10.ef55f8"]]},{"id":"31b5112.2d60fee","type":"ui_group","z":"","name":"detection","tab":"da82a0d.14db76","order":2,"disp":true,"width":"24","collapse":true},{"id":"2a019090.5ba4d","type":"mqtt-broker","z":"","name":"","broker":"192.168.0.240","port":"1883","clientid":"","usetls":false,"compatmode":true,"keepalive":"60","cleansession":true,"birthTopic":"","birthQos":"0","birthPayload":"","closeTopic":"","closePayload":"","willTopic":"","willQos":"0","willPayload":""},{"id":"da82a0d.14db76","type":"ui_tab","z":"","name":"NR Object detection","icon":"dashboard","order":2,"disabled":false,"hidden":false}]
1 Like