Computing an array of 3000 items is to much? I don't think so.
According your first post you have 5 minutes between each message. This should be more than enough to compute the whole array.
It might be wrong but the dangeroust thing is that computing 1 array of about 3000 items in a function node at once, blocks everything else and takes to long. So I think you could measure this duration in milliseconds by:
msg.timeStart = new Date().valueOf();
...your code...
msg.timeEnd = new Date().valueOf();
return msg
I took an example on my engine and there filtering an array of 3000 items took about 1 millisecond:
[
{
"id": "7792bb09a854baa0",
"type": "inject",
"z": "839d658c3816e9e7",
"name": "Timestamp start",
"props": [
{
"p": "topic",
"vt": "str"
},
{
"p": "timeStart",
"v": "",
"vt": "date"
},
{
"p": "timeEnd",
"v": "",
"vt": "date"
}
],
"repeat": "",
"crontab": "",
"once": false,
"onceDelay": 0.1,
"topic": "Laufzeit messen",
"x": 180,
"y": 1440,
"wires": [
[
"d846c10eef3bf71f"
]
]
},
{
"id": "d846c10eef3bf71f",
"type": "function",
"z": "839d658c3816e9e7",
"name": "Array mit 3000 Items",
"func": "msg.topic = \"Messung 1.1\"\nmsg.timeStart = new Date().valueOf();\nmsg.payload = new Array(3000).fill().map((el, idx) => { return {id: idx.toString(), value:Math.random()}});\nmsg.timeEnd = new Date().valueOf();\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 460,
"y": 1440,
"wires": [
[
"a05badbc4295ae2c",
"d289f19cea9dec5f"
]
]
},
{
"id": "a05badbc4295ae2c",
"type": "debug",
"z": "839d658c3816e9e7",
"name": "Array Generated",
"active": false,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 470,
"y": 1400,
"wires": []
},
{
"id": "d289f19cea9dec5f",
"type": "function",
"z": "839d658c3816e9e7",
"name": "Filter Array",
"func": "msg.topic = \"Messung 1.1\"\nmsg.timeStart = new Date().valueOf();\nlet myArray = msg.payload;\nif (context.get('lastArray') !== undefined) {\n myArray = myArray.filter((el, idx) => { \n if (el.value > context.get('lastArray')[idx].value * 1.05) {\n return el;\n }\n });\n}\ncontext.set('lastArray', msg.payload);\nmsg.payload = myArray;\nmsg.timeEnd = new Date().valueOf();\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 810,
"y": 1440,
"wires": [
[
"6b9e12e16a0e114d"
]
]
},
{
"id": "6b9e12e16a0e114d",
"type": "debug",
"z": "839d658c3816e9e7",
"name": "Array Filtered",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 820,
"y": 1400,
"wires": []
}
]
Perhaps you could do the same. As a workaround you could split the computing into chunks of 100 items and this way give node red the choice to compute other nodes between. An example:
[
{
"id": "5bf81ce259910494",
"type": "inject",
"z": "839d658c3816e9e7",
"name": "Timestamp start",
"props": [
{
"p": "topic",
"vt": "str"
},
{
"p": "timeStart",
"v": "",
"vt": "date"
},
{
"p": "timeEnd",
"v": "",
"vt": "date"
}
],
"repeat": "",
"crontab": "",
"once": false,
"onceDelay": 0.1,
"topic": "Laufzeit messen",
"x": 160,
"y": 1780,
"wires": [
[
"6e9696f91d3fc4c1"
]
]
},
{
"id": "6e9696f91d3fc4c1",
"type": "function",
"z": "839d658c3816e9e7",
"name": "Array mit 100000 Items",
"func": "msg.topic = \"Messung 1.1\"\nmsg.timeStart = new Date().valueOf();\nmsg.payload = new Array(10000).fill().map((el, idx) => { return {id: idx.toString(), value:Math.random()}});\nmsg.timeEnd = new Date().valueOf();\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 450,
"y": 1780,
"wires": [
[
"4bb669bcfe173505",
"38a8075ae5618b92"
]
]
},
{
"id": "4bb669bcfe173505",
"type": "function",
"z": "839d658c3816e9e7",
"name": "Messung 1.2 (Split in 100er Chunks)",
"func": "msg.timeStart = new Date().valueOf();\nconst arrPart = []\nlet startIdx;\nlet endIdx;\n\nif(msg.trigger === undefined){\n context.set('lastArray', context.get('array'));\n context.set('array', msg.payload);\n msg.parts = {};\n msg.parts.type = \"array\";\n msg.parts.count = Math.ceil(msg.payload.length/100);\n msg.parts.len = 100;\n msg.parts.index = 0;\n msg.parts.id = msg._msgid;\n msg.arraylength = msg.payload.length;\n startIdx = msg.parts.index;\n}else{\n startIdx = msg.parts.index * 100\n}\n\n//Unterteilung in 100er chunks\nendIdx = startIdx + 100;\nif(msg.arraylength < endIdx){\n endIdx = msg.arraylength;\n msg.parts.len = endIdx - startIdx;\n}\n\nfor(let i=startIdx; i<endIdx; i++){\n //************************************************************\n //HIER DIE AUFGABE, WAS FĂśR JEDEN ITEM GEMACHT WIRD!!!!!!\n if(msg.parts.index==0){\n msg.computeArrayStart = new Date().valueOf();\n }else{\n msg.computeArrayEnd = new Date().valueOf(); \n }\n if(context.get('lastArray')===undefined){\n arrPart.push(context.get(\"array\")[i]);\n }else{\n if (context.get('array')[i].value > context.get('lastArray')[i].value * 1.05) {\n arrPart.push(context.get(\"array\")[i]);\n }\n }\n //************************************************************\n}\nmsg.parts.len = arrPart.length;\n\nmsg.payload = arrPart;\nmsg._msgid = RED.util.generateId();\nmsg.timeEnd = new Date().valueOf();\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 890,
"y": 1780,
"wires": [
[
"375d5a507181218e",
"d67ba09735431465"
]
]
},
{
"id": "375d5a507181218e",
"type": "function",
"z": "839d658c3816e9e7",
"name": "Msg Control (While Do)",
"func": "msg.timeStart = new Date().valueOf();\nif(msg.parts.index < msg.parts.count-1){\n let triggerMsg = RED.util.cloneMessage(msg);\n triggerMsg.trigger = true;\n triggerMsg.parts.index++;\n return[msg, triggerMsg];\n}\nmsg.timeEnd = new Date().valueOf();\nreturn [msg, null];\n\n",
"outputs": 2,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 910,
"y": 1700,
"wires": [
[
"d3a8aa33af2e8fe3",
"84db9fc580780309"
],
[
"4bb669bcfe173505"
]
]
},
{
"id": "d3a8aa33af2e8fe3",
"type": "debug",
"z": "839d658c3816e9e7",
"name": "Chunk Triggered",
"active": false,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 930,
"y": 1660,
"wires": []
},
{
"id": "38a8075ae5618b92",
"type": "debug",
"z": "839d658c3816e9e7",
"name": "Array Generated",
"active": false,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 450,
"y": 1740,
"wires": []
},
{
"id": "d67ba09735431465",
"type": "debug",
"z": "839d658c3816e9e7",
"name": "Chunk Splitted",
"active": false,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 940,
"y": 1840,
"wires": []
},
{
"id": "84db9fc580780309",
"type": "join",
"z": "839d658c3816e9e7",
"name": "",
"mode": "auto",
"build": "object",
"property": "payload",
"propertyType": "msg",
"key": "topic",
"joiner": "\\n",
"joinerType": "str",
"accumulate": "false",
"timeout": "",
"count": "",
"reduceRight": false,
"x": 1230,
"y": 1680,
"wires": [
[
"55fb5e2b92f8dfcc"
]
]
},
{
"id": "55fb5e2b92f8dfcc",
"type": "debug",
"z": "839d658c3816e9e7",
"name": "Computed Array",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 1440,
"y": 1680,
"wires": []
}
]
Another thing you could do is to use https://flows.nodered.org/node/node-red-contrib-mp-function. This node creates a worker thread parallel to your one threaded node red.
The next thing would be to open a worker thread in your browser app and not to compute on server side. Just handle the array over to your GUI and do your work from this side.
