Memory problem Dashboard/plotly.js

Dashboard
1

We are struggling with a memory problem. The attached flow should be ready to run but following nodes are needed:

  • node-red-contrib-linux-memory (0.8.1)
  • node-red-contrib-pid (1.1.7)
  • node-red-dashboard (2.28.1)

To make it easier for somebody to test we are saving the graph data in the default context. Normally, we store this in a persistent context. The data produced by the simulation is not the problem here.

CH2 shows the memory available over time

memory-problem
memory-problem800Ă—800 19.1 KB

Could somebody who is familiar with the dashboard and/or plotly.js have a look at this? If it is not a memory leak, is there a way to change GC settings?

node-red: v1.2.9
node: v12.13.0

[{"id":"b4973237.46ab6","type":"subflow","name":"Process Simulation","info":"","in":[{"x":37,"y":103,"wires":[{"id":"ec719d4d.0d54f8"}]}],"out":[{"x":728.5,"y":294,"wires":[{"id":"ae1a6e5.d4c0d9","port":0}]}]},{"id":"7fe4b5c3.32e58c","type":"function","z":"b4973237.46ab6","name":"30 sec RC + 20","func":"// Applies a simple RC low pass filter to incoming payload values\nvar tc = 30*1000;       // time constant in milliseconds\n\nvar lastValue = context.get('lastValue');\nif (typeof lastValue == \"undefined\") lastValue = msg.payload;\nvar lastTime = context.get('lastTime') || null;\nvar now = new Date();\nvar currentValue = msg.payload;\nif (lastTime === null) {\n    // first time through\n    newValue = currentValue;\n} else {\n    var dt = now - lastTime;\n    var newValue;\n    \n    if (dt > 0) {\n        var dtotc = dt / tc;\n        newValue = lastValue * (1 - dtotc) + currentValue * dtotc;\n    } else {\n        // no time has elapsed leave output the same as last time\n        newValue = lastValue;\n    }\n}\ncontext.set('lastValue', newValue);\ncontext.set('lastTime', now);\n\nmsg.payload = newValue + 30;\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":640,"y":220,"wires":[["ae1a6e5.d4c0d9"]]},{"id":"1bacd004.9753c","type":"inject","z":"b4973237.46ab6","name":"Inject -0.2 at start","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":true,"onceDelay":"","topic":"","payload":"-0.2","payloadType":"num","x":134.5,"y":30,"wires":[["ec719d4d.0d54f8"]]},{"id":"999a52c2.f465f","type":"function","z":"b4973237.46ab6","name":"10 sec RC","func":"// Applies a simple RC low pass filter to incoming payload values\nvar tc = 10*1000;       // time constant in milliseconds\n\nvar lastValue = context.get('lastValue');\nif (typeof lastValue == \"undefined\") lastValue = msg.payload;\nvar lastTime = context.get('lastTime') || null;\nvar now = new Date();\nvar currentValue = msg.payload;\nif (lastTime === null) {\n    // first time through\n    newValue = currentValue;\n} else {\n    var dt = now - lastTime;\n    var newValue;\n    \n    if (dt > 0) {\n        var dtotc = dt / tc;\n        newValue = lastValue * (1 - dtotc) + currentValue * dtotc;\n    } else {\n        // no time has elapsed leave output the same as last time\n        newValue = lastValue;\n    }\n}\ncontext.set('lastValue', newValue);\ncontext.set('lastTime', now);\n\nmsg.payload = newValue;\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":451,"y":207,"wires":[["7fe4b5c3.32e58c"]]},{"id":"ec719d4d.0d54f8","type":"delay","z":"b4973237.46ab6","name":"","pauseType":"delay","timeout":"0.5","timeoutUnits":"seconds","rate":"1","nbRateUnits":"","rateUnits":"second","randomFirst":"1","randomLast":"5","randomUnits":"seconds","drop":false,"x":278,"y":104,"wires":[["ede39236.1961f8"]]},{"id":"a823c9cf.2a6178","type":"function","z":"b4973237.46ab6","name":"2 msg transport delay","func":"// stores messages in a fifo until the specified number have been received, \n// then releases them as new messages are received.\n// during the filling phase the earliest message is passed on each time \n// a message is received, but it is also left in the fifo\nvar fifoMaxLength = 2;\nvar fifo = context.get('fifo') || [];\n// push the new message onto the top of the array, messages are shifted down and\n// drop off the front\nvar length = fifo.push(msg);  // returns new length\nif (length > fifoMaxLength) {\n    newMsg = fifo.shift();\n} else {\n    // not full yet, make a copy of the msg and pass it on\n    var newMsg = JSON.parse(JSON.stringify(fifo[0]));\n}\ncontext.set('fifo', fifo);\nreturn newMsg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":260,"y":220,"wires":[["999a52c2.f465f"]]},{"id":"ae1a6e5.d4c0d9","type":"function","z":"b4973237.46ab6","name":"Clear all except payload","func":"msg2 = {payload: msg.payload};\nreturn msg2;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":545,"y":293,"wires":[[]]},{"id":"ede39236.1961f8","type":"range","z":"b4973237.46ab6","minin":"0","maxin":"1","minout":"0","maxout":"100","action":"scale","round":false,"property":"payload","name":"","x":87,"y":208,"wires":[["a823c9cf.2a6178"]]},{"id":"de69eea5.98a61","type":"tab","label":"Flow 2","disabled":false,"info":""},{"id":"22d8f7bd.d97cd8","type":"group","z":"de69eea5.98a61","name":"Advanced Graph (plotly.js)","style":{"label":true},"nodes":["63945869.2a8a98","e3203b44.f64818","eb258cd7.3839a","b600f37c.5480b","c7eea535.cff998","f4146d0b.fff29","f1a547a.b1f0ab8","1c73b5cf.df6eba","885d2b36.e12938","f91e30da.68a9f","c5a45422.05fd78","fd090c29.e8dbe","b9711f03.ff45a","ab2c716f.05ff","3afdf86a.6df9f8","ecb6e8fa.e19b48","51f79710.27e168","8ac1866f.768cc8","398431c6.2b15fe","6b75c3b0.f4e59c","1f729ea.68a5a61"],"x":1014,"y":79,"w":1032,"h":442},{"id":"75a219a6.3e67b8","type":"group","z":"de69eea5.98a61","name":"Memory monitoring","style":{"label":true},"nodes":["c113aa4b.85b348","d0485aca.231e38","c35ca5a4.4b5ec8","78297ac2.918984"],"x":94,"y":499,"w":812,"h":82},{"id":"846c946a.aae9b8","type":"group","z":"de69eea5.98a61","name":"PID Simulation","style":{"label":true},"nodes":["e0cb2a8c.4402a8","f70949c8.aab988","2792f805.066c78","1bf3ce16.c3ae02","f69bbf05.bdd49","143523b2.2f165c","5ec4239b.34e174","de738fc5.75e4c8","e94f888a.130a88","bef0a1f8.2e2cf","9079ffaf.4a096","f6889eca.85c47","e38e1f3a.72b78"],"x":74,"y":87,"w":852,"h":317},{"id":"e0cb2a8c.4402a8","type":"PID","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"","setpoint":"50","pb":"6","ti":"16","td":"4","integral_default":"0","smooth_factor":"0","max_interval":600,"enable":"1","disabled_op":"0","x":450,"y":220,"wires":[["5ec4239b.34e174","e94f888a.130a88","e38e1f3a.72b78"]]},{"id":"f70949c8.aab988","type":"change","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"live data","tot":"str"},{"t":"set","p":"channel_nbr","pt":"msg","to":"1","tot":"num"}],"action":"","property":"","from":"","to":"","reg":false,"x":820,"y":280,"wires":[["1f729ea.68a5a61"]]},{"id":"2792f805.066c78","type":"inject","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"Setpoint 100","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":"","topic":"setpoint","payload":"100","payloadType":"num","x":192,"y":243,"wires":[["e0cb2a8c.4402a8"]]},{"id":"1bf3ce16.c3ae02","type":"inject","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"Setpoint 150","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":"","topic":"setpoint","payload":"150","payloadType":"num","x":190.5,"y":292,"wires":[["e0cb2a8c.4402a8"]]},{"id":"f69bbf05.bdd49","type":"inject","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"enable","repeat":"","crontab":"","once":false,"topic":"enable","payload":"true","payloadType":"bool","x":183,"y":128,"wires":[["e0cb2a8c.4402a8"]]},{"id":"143523b2.2f165c","type":"inject","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"disable","repeat":"","crontab":"","once":false,"topic":"enable","payload":"false","payloadType":"bool","x":183.5,"y":178,"wires":[["e0cb2a8c.4402a8"]]},{"id":"5ec4239b.34e174","type":"subflow:b4973237.46ab6","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"","env":[],"x":453,"y":147,"wires":[["de738fc5.75e4c8","e0cb2a8c.4402a8"]]},{"id":"de738fc5.75e4c8","type":"change","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"live data","tot":"str"},{"t":"set","p":"channel_nbr","pt":"msg","to":"0","tot":"num"}],"action":"","property":"","from":"","to":"","reg":false,"x":820,"y":140,"wires":[["1f729ea.68a5a61"]]},{"id":"e94f888a.130a88","type":"range","z":"de69eea5.98a61","g":"846c946a.aae9b8","minin":"0","maxin":"1","minout":"0","maxout":"100","action":"scale","round":false,"property":"payload","name":"Scale power","x":515,"y":295,"wires":[["f70949c8.aab988"]]},{"id":"bef0a1f8.2e2cf","type":"inject","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"Clear chart on deploy","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":true,"onceDelay":"","topic":"","payload":"{\"data\":[]}","payloadType":"json","x":440,"y":363,"wires":[["9079ffaf.4a096"]]},{"id":"9079ffaf.4a096","type":"change","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"","rules":[{"t":"move","p":"payload.data","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":676,"y":363,"wires":[[]]},{"id":"f6889eca.85c47","type":"inject","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"Setpoint 80","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":"","topic":"setpoint","payload":"200","payloadType":"num","x":190,"y":340,"wires":[["e0cb2a8c.4402a8"]]},{"id":"c113aa4b.85b348","type":"inject","z":"de69eea5.98a61","g":"75a219a6.3e67b8","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"1","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"true","payloadType":"bool","x":190,"y":540,"wires":[["d0485aca.231e38"]]},{"id":"d0485aca.231e38","type":"memory","z":"de69eea5.98a61","g":"75a219a6.3e67b8","name":"","relativeValues":false,"unitType":"mb","totalMemory":false,"usedMemory":false,"freeMemory":true,"availableMemory":false,"activeMemory":false,"buffersMemory":false,"cachedMemory":false,"slabMemory":false,"buffcacheMemory":false,"freeAvailableMemory":false,"swapTotalMemory":false,"swapUsedMemory":false,"swapFreeMemory":false,"x":370,"y":540,"wires":[["78297ac2.918984"]]},{"id":"c35ca5a4.4b5ec8","type":"ui_text","z":"de69eea5.98a61","g":"75a219a6.3e67b8","group":"6aa20356.9b59bc","order":1,"width":0,"height":0,"name":"","label":"RAM","format":"{{msg.payload}} MB","layout":"row-spread","x":830,"y":540,"wires":[]},{"id":"e38e1f3a.72b78","type":"change","z":"de69eea5.98a61","g":"846c946a.aae9b8","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"live data","tot":"str"},{"t":"set","p":"channel_nbr","pt":"msg","to":"3","tot":"num"},{"t":"set","p":"payload","pt":"msg","to":"setpoint","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":820,"y":220,"wires":[["1f729ea.68a5a61"]]},{"id":"63945869.2a8a98","type":"ui_template","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","group":"6aa20356.9b59bc","name":"Load plotly.js","order":9,"width":0,"height":0,"format":"<script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>","storeOutMessages":true,"fwdInMessages":true,"resendOnRefresh":true,"templateScope":"global","x":1570,"y":160,"wires":[[]]},{"id":"e3203b44.f64818","type":"ui_template","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","group":"6aa20356.9b59bc","name":"div for plot","order":3,"width":"30","height":"25","format":"<div id=\"graph-div\" style=\"width:800px; height:800px;\"></div>\n\n","storeOutMessages":false,"fwdInMessages":false,"resendOnRefresh":false,"templateScope":"local","x":1770,"y":160,"wires":[[]]},{"id":"eb258cd7.3839a","type":"ui_template","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","group":"6aa20356.9b59bc","name":"streaming","order":5,"width":"0","height":"0","format":"<script>\n\n(function(scope){\n    \n    \nscope.$watch('msg', function(msg) {\n    if(msg){\n        try{\n            //console.log(msg.topic + msg.payload );\n            \n            // load data from memory\n            if(msg.topic == \"measurement data\"){\n                \n                \n             //if(msg.payload[0].length > 0 ){\n                console.log(\"load data from memory: \"+msg.channel_nbr);\n                Plotly.addTraces('graph-div', {\n                    //type: \"scattergl\",\n                    mode: \"line\",\n                    x: msg.payload[0],\n                    y: msg.payload[1],\n                    name: \"CH\"+msg.channel_nbr\n                }, msg.channel_nbr);  // first trace is 0             \n             //}\n    \n             \n             \n            \n            }else if(msg.topic == \"live data\"){\n                \n               /* var minuteView = { // todo: delete\n                    xaxis: {\n                        type: 'date'\n                    }\n                };*/\n                //console.log(\"live data channel: \"+msg.channel_nbr);\n                Plotly.extendTraces('graph-div', {\n                    x: [[msg.timestamp]],\n                    y: [[msg.payload]]\n                }, \n                    \n                [msg.channel_nbr]); // channel number, has to be an index!\n                \n                \n            }else if(msg.topic == \"restyle\"){\n                console.log(\"restyle: \"+msg.channel_nbr);\n                \n                Plotly.restyle('graph-div', msg.update, [msg.channel_nbr]);\n            }\n            \n        }catch{\n            \n        }\n    }\n    \n    \n    //msg = undefined;\n\n    \n});\n\n    //scope = null; // prevent memory leak\n\n\n})(scope);\n\n\n</script>\n","storeOutMessages":false,"fwdInMessages":true,"resendOnRefresh":false,"templateScope":"local","x":1960,"y":400,"wires":[[]]},{"id":"b600f37c.5480b","type":"function","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"live data","func":"\n    if(msg.topic === \"live data\" ){\n        \n        flow.set(\"live_data_counter\",flow.get(\"live_data_counter\")+1);\n        \n        // create and format timestamp\n        msg.timestamp = new Date();\n        datestring = msg.timestamp.getFullYear()+\"-\"+(msg.timestamp.getMonth()+1)+\"-\"+msg.timestamp.getDate();\n        timestring = msg.timestamp.toLocaleTimeString('de-CH',\n        { hour: '2-digit',\n          minute: '2-digit',\n          second: '2-digit',\n          \n        }\n        \n        );  \n        \n        msg.timestamp = datestring+\" \"+timestring+\".\"+msg.timestamp.getMilliseconds().toString().padStart(3,\"0\"); // leading zeros probably working with newer node.js?\n        \n        \n        let measurement_data = flow.get(\"measurement_data\",\"default\");\n        \n        // check if channel was initialized (channel_nbr equals index of array element)\n        if(msg.channel_nbr > measurement_data.length-1){\n            measurement_data.push([[],[]]);\n        }\n        \n        // store current measurement in memory\n        measurement_data[msg.channel_nbr][0].push(msg.timestamp);\n        measurement_data[msg.channel_nbr][1].push(msg.payload);\n\n    }\n    \n    if(msg.topic == \"live data switch\"){\n        flow.set(\"live_data_switch\",flow.get(\"live_data_switch\")*-1);\n    }\n    \n    \n    // only send live data to front end if it's open\n    if(flow.get(\"plot_init_done\") && flow.get(\"live_data_switch\") == 1){\n        return msg;\n    }\n    \nreturn;\n","outputs":1,"noerr":0,"initialize":"","finalize":"","x":1620,"y":400,"wires":[["eb258cd7.3839a"]]},{"id":"c7eea535.cff998","type":"ui_template","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","group":"6aa20356.9b59bc","name":"Trigger: Tab with graph loaded","order":7,"width":0,"height":0,"format":"<script>\n\n//(function(scope) {\n    \n //   scope.$watch('msg', function(msg) {\n        \n    \n        console.log(\"frontend init\");\n        //console.log(scope);\n        //let data = [    ];\n        \n        \n        //let layout = {title: \"\", xaxis: {type: 'date'} };\n        //let config = {displaylogo: false, displayModeBar: true, scrollZoom: true};\n        \n        Plotly.newPlot(\n            'graph-div', \n            [], \n            {title: \"\", xaxis: {type: 'date'} }, \n            {displaylogo: false, displayModeBar: true, scrollZoom: true} \n        );\n        \n        //scope.send({payload: \"preload done\"}); // this gets sent when the view is opened in the browser\n        this.scope.send({payload: \"preload done\"});\n        \n        //scope = null; // prevent memory leak\n//    });\n    \n     \n        \n        \n        \n//})(scope);\n\n</script>","storeOutMessages":false,"fwdInMessages":false,"resendOnRefresh":false,"templateScope":"local","x":1170,"y":360,"wires":[["398431c6.2b15fe"]],"info":"This is triggered only in the selected tab (group), therefore no scope.$watch is needed\nscope.send(...) is needed, or else no message object is passed on to the next funciton node to load data for the graph"},{"id":"f4146d0b.fff29","type":"inject","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"},{"p":"tuning","v":"true","vt":"bool"}],"repeat":"","crontab":"","once":true,"onceDelay":"","topic":"","payload":"","payloadType":"date","x":1150,"y":160,"wires":[["f1a547a.b1f0ab8"]]},{"id":"f1a547a.b1f0ab8","type":"function","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"init","func":"nbr_of_channels = 4-1;\ntmp_array = [];\nfor(c = 0; c <= nbr_of_channels; c++){\n    tmp_array.push([[],[]]);\n}\nflow.set(\"measurement_data\",tmp_array,\"default\");\n    \nflow.set(\"plot_init_done\", false);\n    \n// analysis\nflow.set(\"live_data_counter\", 0);\nflow.set(\"live_data_switch\", 1);\nflow.set(\"advanced_graph_on\",1);\n\nnode.warn(\"plotly init done\");\n\nreturn;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":1310,"y":160,"wires":[[]]},{"id":"1c73b5cf.df6eba","type":"ui_ui_control","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","events":"all","x":1120,"y":260,"wires":[["c5a45422.05fd78"]]},{"id":"885d2b36.e12938","type":"comment","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"detect browser interaction","info":"","x":1170,"y":220,"wires":[]},{"id":"f91e30da.68a9f","type":"comment","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"load plotly library and set div container","info":"","x":1650,"y":120,"wires":[]},{"id":"c5a45422.05fd78","type":"switch","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","property":"payload","propertyType":"msg","rules":[{"t":"eq","v":"lost","vt":"str"}],"checkall":"false","repair":false,"outputs":1,"x":1270,"y":260,"wires":[["fd090c29.e8dbe"]]},{"id":"fd090c29.e8dbe","type":"change","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","rules":[{"t":"set","p":"plot_init_done","pt":"flow","to":"false","tot":"bool"}],"action":"","property":"","from":"","to":"","reg":false,"x":1500,"y":260,"wires":[[]]},{"id":"b9711f03.ff45a","type":"comment","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"init measurement data array","info":"","x":1180,"y":120,"wires":[]},{"id":"ab2c716f.05ff","type":"ui_button","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","group":"6aa20356.9b59bc","order":1,"width":0,"height":0,"passthru":false,"label":"live data on/off","tooltip":"","color":"","bgcolor":"","icon":"","payload":"","payloadType":"str","topic":"topic","topicType":"msg","x":1120,"y":400,"wires":[["3afdf86a.6df9f8"]]},{"id":"3afdf86a.6df9f8","type":"change","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"live data switch","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":1340,"y":400,"wires":[["b600f37c.5480b"]]},{"id":"ecb6e8fa.e19b48","type":"function","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","func":"flow.set(\"advanced_graph_on\",flow.get(\"advanced_graph_on\")*-1,\"default\");\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":1860,"y":220,"wires":[[]]},{"id":"51f79710.27e168","type":"inject","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"advanced graph on/off","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":1620,"y":220,"wires":[["ecb6e8fa.e19b48"]]},{"id":"8ac1866f.768cc8","type":"split","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":1610,"y":360,"wires":[["6b75c3b0.f4e59c"]]},{"id":"398431c6.2b15fe","type":"change","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","rules":[{"t":"set","p":"plot_init_done","pt":"flow","to":"true","tot":"bool"},{"t":"set","p":"payload","pt":"msg","to":"measurement_data","tot":"flow"}],"action":"","property":"","from":"","to":"","reg":false,"x":1440,"y":360,"wires":[["8ac1866f.768cc8"]]},{"id":"6b75c3b0.f4e59c","type":"change","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"measurement data","tot":"str"},{"t":"set","p":"channel_nbr","pt":"msg","to":"parts.index","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":1780,"y":360,"wires":[["eb258cd7.3839a"]]},{"id":"1f729ea.68a5a61","type":"switch","z":"de69eea5.98a61","g":"22d8f7bd.d97cd8","name":"","property":"advanced_graph_on","propertyType":"flow","rules":[{"t":"eq","v":"1","vt":"num"}],"checkall":"true","repair":false,"outputs":1,"x":1090,"y":480,"wires":[["b600f37c.5480b"]]},{"id":"78297ac2.918984","type":"change","z":"de69eea5.98a61","g":"75a219a6.3e67b8","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"live data","tot":"str"},{"t":"set","p":"channel_nbr","pt":"msg","to":"2","tot":"num"}],"action":"","property":"","from":"","to":"","reg":false,"x":580,"y":540,"wires":[["1f729ea.68a5a61","c35ca5a4.4b5ec8"]]},{"id":"6aa20356.9b59bc","type":"ui_group","name":"examples","tab":"d5ad8b85.f5ba18","order":1,"disp":false,"width":"30","collapse":false},{"id":"d5ad8b85.f5ba18","type":"ui_tab","name":"Advanced Charts","icon":"dashboard","disabled":false,"hidden":false}]
2

In the future please include your flow in the thread. In order to make code more readable and importable it is important to surround your code with three backticks

`like this`

You can edit your post by clicking the pencil icon to see how I added your flow.

See this post for more details - How to share code or flow json

It is also helpful in getting help, it you provide the full name of any contrib nodes you are using.

what is the 'Memory' node used in the sub flow - full name please.

3

Restart node red and then run, in a terminal,
top and screenshot the top section showing the memory usage, and look for node-red or node in the process table and record what it says for then. Then wait till your plot does low memory and repeat the exercise, then post the results here. That should let us see what is really going on.

Edit Also if you leave it does something eventually fail or does it keep going?
Which version of node-red-dashboard are you using?

4

Thanks @Colin, please see results below. If I leave the system running it goes down to about 6 MB but it doesn't really crash. The used versions you will find in the initial post.

Memory usage right after node-red has been started (sorry it’s in German):

memory-start
memory-start710Ă—127 94.4 KB

Memory usage after about 10 minutes:
memory-after-10mins

Memory usage after about 30 minutes:
memory-after-30mins

Memory curve:

memory-after-30mins
memory-after-30mins800Ă—800 8.77 KB

In the meantime, I did run some more tests on a VM with more memory (approx. 4 Gigs).

memory-tests-VM-4GB
memory-tests-VM-4GB800Ă—800 10.6 KB

The GC seems to free some memory occasionally. The node version is even older (v10.19.0) but the linux system is newer Ubuntu (20.04.1 LTS). The other components do have the same version as the productive system (where the mentioned memory problems occur).

5

What device and OS are you running on? Is it a Pi3 and Raspbian?

What value have you got for max_old_space_size when you run node-red? If you used the recommended install script for raspbian/debian/ubuntu, and are using systemd to run node red then that will be setup in /lib/systemd/system/nodered.service

The fact that it doesn't actually crash means that you haven't got a memory leak, but obviously node-red is doing something pretty memory intensive. Have you checked the situation if you remove the charting stuff?
I am not able to run your flow at the moment, how often are you adding points to the chart and what time range does it show?
I presume that you know that it is not the memory charting that is the problem.

6

I have most likely found the node which causes the problem (it's our code :see_no_evil:) but I don't fully understand which line(s) cause the leak. Before a data point is sent to plotly, we create a timestamp and format it. The timestamp is then stored as a string so I believe that the date object will be deleted sooner or later. Even tried to set things to undefined once the timestamp is stored in the array - didn't help. If I use just an index as x-axis (no timestamp), the system runs very stable (tested 4 hours so far).

The leak must be somewhere in the code below.

if(msg.topic === "live data" ){
        
    flow.set("live_data_counter",flow.get("live_data_counter")+1);
        
    // create and format timestamp
    let current_timestamp = new Date();
    let datestring = current_timestamp.getFullYear()+"-"+(current_timestamp.getMonth()+1)+"-"+current_timestamp.getDate();
        
    let timestring = current_timestamp.toLocaleTimeString('de-CH',
    { hour: '2-digit',
      minute: '2-digit',
      second: '2-digit',
    });  
        
    
    let measurement_data = flow.get("measurement_data","default");
        
    // check if channel was initialized (channel_nbr equals index of array element)
    if(msg.channel_nbr > measurement_data.length-1){
        measurement_data.push([[],[]]);
    }
        
    // store current measurement in memory
    msg.timestamp = datestring+" "+timestring+"."+current_timestamp.getMilliseconds().toString().padStart(3,"0");
    measurement_data[msg.channel_nbr][0].push(msg.timestamp);
    measurement_data[msg.channel_nbr][1].push(Math.round(msg.payload));
    
    
    
    /*current_timestamp = undefined;
    datestring = undefined;
    timestring = undefined;*/
    

}
    
if(msg.topic == "live data switch"){
    flow.set("live_data_switch",flow.get("live_data_switch")*-1);
}

// only send live data to front end if it's open
if(flow.get("plot_init_done") && flow.get("live_data_switch") == 1){
    return msg;
}

return;
7

datestring and timestring are local to the block so may be released as soon as you drop out of the block I don't know how local strings are allocated in javascript, the memory might not be released until garbage collection, or it may be released immediately. More to the point are the timestamp strings that you create and add to measurement_data which exists in the flow context, also I see you are adding the string to msg.timestamp which means it will also be passed on with the message. Where those are released depends on what you do with the passed on message and what happens to the measurement_data structure in context.

However, if the system is not crashing then you have not got a memory leak, you are just using a lot of memory. Did you see the questions in my previous post?

8

Thanks again @Colin ..okay, let me try to answer all the questions:

We get these controllers preconfigured from the manufacturer (it's not a Pi, our system has 512 MB RAM). If I look at the service config file, it doesn't seem to have a max_old_space_size set. How or where can I check what size parameter is being used?

It probably didn't crash because the SWAP is not full yet?

As mentioned in the previous comment, the system (with charting on) runs stable if the flow doesn't create timestamps.

The simulation sends data every 500 ms. The productive system will send data approx. every 200 ms.

The data is stored in a file (persistent context). I changed this to default context, so it should be easier for people to test (the amount of "measurement data" is not growing as fast as the memory drops).

The msg.timestamp is needed in the ui_template which is the last node. Do I have to destroy the msg manually?:

<script>

(function(scope){

scope.$watch('msg', function(msg) {
    if(msg){
        try{
            //console.log(msg.topic + msg.payload );
            
            // load data from memory
            if(msg.topic == "measurement data"){
                
                console.log("load data from memory: "+msg.channel_nbr);
                Plotly.addTraces('graph-div', {
                    type: "scatter",
                    mode: "line",
                    x: msg.payload[0],
                    y: msg.payload[1],
                    name: "CH"+msg.channel_nbr
                }, msg.channel_nbr);  // first trace is 0             

            }else if(msg.topic == "live data"){
                
                //console.log("live data channel: "+msg.channel_nbr);
                Plotly.extendTraces('graph-div', {
                    x: [[msg.timestamp]],
                    y: [[msg.payload]]
                }, 
                    
                [msg.channel_nbr]); // channel number, has to be an index!
                
                
            }else if(msg.topic == "restyle"){
                console.log("restyle: "+msg.channel_nbr);
                
                Plotly.restyle('graph-div', msg.update, [msg.channel_nbr]);
            }
            
        }catch{
            
        }
    }
});

})(scope);

</script>
9

That is probably why node-red is using up all the memory before garbage collecting. For a system with 512MB you should possibly set it to 256. Then node red will release memory when it gets to that size. See how it goes then.

10

No, you don't need to destroy anything manually, once it gets to the point that it is no longer referenced anywhere then it will be available for release.
The context data exists in a cache in memory even if it is file backed. If, for example, you empty the array then everything in will be released (unless anything in there is referenced elsewhere). Presumably the array does not keep growing for ever.

11

Do you really need the locale timestamp there? Could you not use an ISO string instead? My understanding is that the date/time locale handling of JavaScript in general is quite slow and heavyweight. Converting to an ISO string doesn't loose any information and is serialisable.

Alternatively, maybe consider keeping your date/time stamp as either a JS or UNIX timestamp number.

12

Oh wow. @TotallyInformation creating timestamps with toISOString() looks very promising. I let it run for some time and let you know if it's stable. Just need to convert it to local time but that should not be a big deal.

And also thank you @Colin for all the valuable information.

13

As a general rule, it is best to leave timestamps in UTC until you display it. That way you never have to worry about DST and locale issues when calculating and storing things.

14

Sadly you still do not know the real cause of the problem. It would have been better to try setting the memory limit first in order to check that is the real cause of node-red consuming all the memory. Otherwise if you develop the system further and node-red needs more memory you will hit it again.

Out of personal interest are you developing an embedded PID controller using node-red and the dashboard for display? I am the author of that node.

15

We are still testing. I tried to change the memory limit first indeed but it did not have the expected effect.
I added the following line to the service file:

[Service]
Environment="NODE_OPTIONS=--max_old_space_size=256"

Since persistent context is somewhere in the memory still (which I did not know), we have to calculate the size of this array. The memory is still growing but it's way better with:

msg.timestamp = new Date().toISOString().replace("T"," ").replace("Z","");

Yes we are planning to use the PID to control an induction heating process. So far we did some simple tests with a stove in our lab:

pid-test-2
pid-test-2800Ă—800 20.8 KB

This week we will try the PID on a real induction heating system. The temperature measurement is done with a pyrometer (contactless infrared measurement).

16

Does $NODE_OPTIONS appear in the ExecStart line?
Did it have any effect? It should have stopped node red from consuming more than 256MB. Which could have crashed node-red if it actually needed more than that, or would just have limited it at that as the garbage collection freed up available memory.

I don't know exactly what you are doing with the array but if it is going to keep growing you might be better with a database such as sqlite. Normally I would suggest Influxdb but that may not be appropriate for your system, depending on what file storage device it has. If you want to save even more space for the timestamp keep it as a milliseconds value and convert it when you need it.
msg.timestamp = new Date().getTime()

Excellent. It is good to know what ones work is being used for. Thanks.

17

That's not how max old space works I'm afraid. All it does it provide a hint to the V8 engine to do GC at that level of stack use. It doesn't change the total amount of memory available to Node.js.

That in itself will help of course but only in very specific circumstances which is why I never both with it and haven't since I stopped using my original Pi1 - not sure it really helped even then.

Earlier GC will tend to keep a little more stack space available to node.js which is why it can delay crashes due to out of stack problems but that is all it does as far as I am aware.

18

Are you sure, I haven't seen that documented anywhere?

19

Yes, it is around somewhere. You can test it for yourself by setting max old space to a low value then printing out the stack space from within node-red.

I keep this in my settings.js file just before the module.exports line:

/** Optionally display the Node.js/v8 engine heap size on startup */
const v8 = require('v8')
console.info(`V8 Total Heap Size: ${(v8.getHeapStatistics().total_available_size / 1024 / 1024).toFixed(2)} MB`)

On my 16GB dev PC, it shows 4128.85 MB

20

Well the official docs are here - Command-line options | Node.js v15.9.0 Documentation
but as I understand it @TotallyInformation is correct - it is a setting to which the garbage collector will try to work... of course it can only collect garbage/unused objects - so if you actually have live objects (ie lots of data being processed or indeed several large objects like images/files etc) then it can't collect/remove those and things will go bang anyway. As also noted the default size is 4GB.