# While Passing Array to Node and Reformating Array into an Object Data Gets Lost
**URL:** https://discourse.nodered.org/t/while-passing-array-to-node-and-reformating-array-into-an-object-data-gets-lost/14005
**Category:** General
**Created:** [5 August 2019 02:17 UTC](https://discourse.nodered.org/t/while-passing-array-to-node-and-reformating-array-into-an-object-data-gets-lost/14005 "2019-08-05T02:17:09Z")
**Posts on this page:** 4
**Page:** 1
### Author:  [@annibauer](https://discourse.nodered.org/u/annibauer)
#### Post date: [5 August 2019 02:17 UTC](https://discourse.nodered.org/t/while-passing-array-to-node-and-reformating-array-into-an-object-data-gets-lost/14005/1 "2019-08-05T02:17:09Z")
I am extracting data from a CSV file, that I want to send to a server through an API Call. In order to limit the number of calls I'm formatting the request body to contain several data points at once. All this is working fine, besides the fact that only a fraction of the data is arriving at the server. I have debugged the problem and noticed that from one node to the next lies the problem.
I pass an array full of data points to a node and add this array to the body object for the API request. I pass this object on to the request node. When consoling the object only 1000 of the data points appear although a couple thousand were sent into the node. These were also received, but within the object not passed on.
All this is happening within the "Request Body Formatting" node.
I'd really appreciate any help! 🙂
I've added the flow below:
[{"id":"947cab1a.0d5bf","type":"tab","label":"Flow 5","disabled":false,"info":""},{"id":"65c95823.a10ae","type":"http request","z":"947cab1a.0d5bf","name":"","method":"POST","ret":"obj","paytoqs":true,"url":"[http://192.168.183.2:8800/ppm/v2/measurement/","tls":"","proxy":"","authType":"basic","x":530,"y":300,"wires":[["f98141a.60f2a4"]]},{"id":"f98141a.60f2a4","type":"debug","z":"947cab1a.0d5bf","name":"","active":false,"tosidebar":true,"console":true,"tostatus":false,"complete":"true","targetType":"full","x":710,"y":300,"wires":[]},{"id":"b6e0592a.23c17","type":"inject","z":"947cab1a.0d5bf","name":"","topic":"","payload":"","payloadType":"date","repeat":"","crontab":"","once":false,"onceDelay":0.1,"x":100,"y":120,"wires":[["55e126aa.826ec"]]},{"id":"ba311656.326d98","type":"csv","z":"947cab1a.0d5bf","name":"","sep":",","hdrin":true,"hdrout":"","multi":"one","ret":"\n","temp":"","skip":"0","x":870,"y":120,"wires":[["82139ff5.5695f"]]},{"id":"9b1fe86b.85791","type":"file](http://192.168.183.2:8800/ppm/v2/measurement/%22,%22tls%22:%22%22,%22proxy%22:%22%22,%22authType%22:%22basic%22,%22x%22:530,%22y%22:300,%22wires%22:%5B%5B%22f98141a.60f2a4%22%5D%5D%7D,%7B%22id%22:%22f98141a.60f2a4%22,%22type%22:%22debug%22,%22z%22:%22947cab1a.0d5bf%22,%22name%22:%22%22,%22active%22:false,%22tosidebar%22:true,%22console%22:true,%22tostatus%22:false,%22complete%22:%22true%22,%22targetType%22:%22full%22,%22x%22:710,%22y%22:300,%22wires%22:%5B%5D%7D,%7B%22id%22:%22b6e0592a.23c17%22,%22type%22:%22inject%22,%22z%22:%22947cab1a.0d5bf%22,%22name%22:%22%22,%22topic%22:%22%22,%22payload%22:%22%22,%22payloadType%22:%22date%22,%22repeat%22:%22%22,%22crontab%22:%22%22,%22once%22:false,%22onceDelay%22:0.1,%22x%22:100,%22y%22:120,%22wires%22:%5B%5B%2255e126aa.826ec%22%5D%5D%7D,%7B%22id%22:%22ba311656.326d98%22,%22type%22:%22csv%22,%22z%22:%22947cab1a.0d5bf%22,%22name%22:%22%22,%22sep%22:%22,%22,%22hdrin%22:true,%22hdrout%22:%22%22,%22multi%22:%22one%22,%22ret%22:%22%5Cn%22,%22temp%22:%22%22,%22skip%22:%220%22,%22x%22:870,%22y%22:120,%22wires%22:%5B%5B%2282139ff5.5695f%22%5D%5D%7D,%7B%22id%22:%229b1fe86b.85791%22,%22type%22:%22file) in","z":"947cab1a.0d5bf","name":"","filename":"C:\Users\ABN1LR\Desktop\weldingDataFilter.csv","format":"utf8","chunk":false,"sendError":false,"encoding":"none","x":590,"y":120,"wires":[["ba311656.326d98"]]},{"id":"82139ff5.5695f","type":"function","z":"947cab1a.0d5bf","name":"Topic Added","func":"msg.topic = msg.payload.timerName;\nreturn msg;","outputs":1,"noerr":0,"x":250,"y":200,"wires":[["c5e4f18c.4084e8"]]},{"id":"f9b9e089.404b3","type":"function","z":"947cab1a.0d5bf","name":"Data Processing","func":"full\_batch = msg.payload;\nspatter\_running\_total = 0;\n\nvar spatter\_rate\_history = flow.get('spatter\_rate\_history') || {};\nvar spatter\_running\_total = flow.get('spatter\_rate\_running\_total') || {};\nvar spatter\_count = flow.get('spatter\_count') || {};\n\nvar body\_array = \n\nfor (var message\_id in full\_batch){\n \n if((full\_batch[message\_id]["stabilisationFactorActValue"] !== 0) ){\n\n timerName = full\_batch[message\_id]["timerName"];\n time = full\_batch[message\_id].dateTime;\n time\_split = time.split("/");\n time\_split\_time = time\_split[2].split(" ");\n time\_split\_hour\_minute = time\_split\_time[1].split(":");\n time\_split\_hour = "0"+ time\_split\_hour\_minute[0];\n hour = time\_split\_hour.substring(time\_split\_hour.length-2, time\_split\_hour.length+1);\n time\_split\_minute = "0"+ time\_split\_hour\_minute[1];\n minute = time\_split\_minute.substring(time\_split\_minute.length-2, time\_split\_minute.length+1);\n var month = "03" //"0"+ time\_split[0];\n month = month.substring(month.length-2, month.length+1);\n var day = "0"+ (parseInt(time\_split[1])-15);\n day = day.substring(day.length-2, day.length+1);\n var year = "2019" //"20" + time\_split\_time[0];\n \n time\_formatted = year + "-" + month + "-" + day + "T" + hour + ":" + minute + ":00Z";\n \n if (timerName in spatter\_running\_total){\n if (full\_batch[message\_id]["uirExpulsionTime"] !== 0){\n spatter\_running\_total[timerName] +=1;\n }\n }\n else{\n if (full\_batch[message\_id]["uirExpulsionTime"] !== 0){\n spatter\_running\_total[timerName] =1;\n }\n spatter\_running\_total[timerName] = 0;\n }\n \n if (timerName in spatter\_count){\n spatter\_count[timerName] +=1;\n }\n else{\n spatter\_count[timerName] = 1;\n }\n \n body\_array.push(\n {\n "ts": time\_formatted,\n "result": "OK",\n "series": {\n "$\_time": [\n 0\n],\n "PSF": [\n full\_batch[message\_id].stabilisationFactorActValue\n ],\n "UIP":[\n full\_batch[message\_id].uipActualValue\n ],\n "Spatter Rate": [\n spatter\_running\_total[timerName]/spatter\_count[timerName]\n ],\n "Voltage": [\n full\_batch[message\_id].voltageActualValue\n ],\n "Current":[\n full\_batch[message\_id].currentActualValue\n ],\n "uirExpulsionTime":[\n full\_batch[message\_id].uirExpulsionTime\n ]\n }\n }\n \n );\n \n if (timerName in spatter\_rate\_history){\n spatter\_rate\_history[timerName].push(spatter\_running\_total[timerName]/spatter\_count[timerName]);\n }\n else{\n spatter\_rate\_history[timerName] = \n spatter\_rate\_history[timerName].push(spatter\_running\_total[timerName]/spatter\_count[timerName]);\n }\n }\n \n}\n\nflow.set('spatter\_rate\_running\_total', spatter\_running\_total);\nflow.set('spatter\_count',spatter\_count);\nflow.set('spatter\_rate\_history', spatter\_rate\_history);\n\nmsg.payload = body\_array;\n\nreturn msg;\n\n\n","outputs":1,"noerr":0,"x":720,"y":200,"wires":[["911a10da.a93a58"]]},{"id":"911a10da.a93a58","type":"function","z":"947cab1a.0d5bf","name":"Request Body Formatting","func":"// MSG.PAYLOAD HOLDS THE DATA POINT ARRAY\n// when consoling the whole msg.payload from this node there is always a limit of 1000 data points\ncounter\_test = msg.payload.length;\n \n msg.payload = {\n "content-spec": "urn:spec://eclipse.org/unide/measurement-message#v2",\n "device": {\n "deviceID": msg.topic\n },\n "measurements": msg.payload\n} ;\n\nmsg.headers = {'content-type':'application/json'};\nmsg.counter = counter\_test;\nreturn msg;\n\n\n","outputs":1,"noerr":0,"x":270,"y":300,"wires":[["8be76cd5.757438"]]},{"id":"55e126aa.826ec","type":"function","z":"947cab1a.0d5bf","name":"Reset flow variables","func":"var spatter\_rate\_history = flow.get('spatter\_rate\_history') || {};\nvar spatter\_running\_total = flow.get('spatter\_rate\_running\_total') || {};\nvar spatter\_count = flow.get('spatter\_count') || {};\n\nflow.set('spatter\_rate\_running\_total', {});\nflow.set('spatter\_count', {});\nflow.set('spatter\_rate\_history', {});\n\nreturn msg;","outputs":1,"noerr":0,"x":280,"y":120,"wires":[["9b1fe86b.85791"]]},{"id":"c5e4f18c.4084e8","type":"batcher","z":"947cab1a.0d5bf","name":"","maxTopics":"5","maxMessagesPerTopic":"500000","maxDelay":"1000","x":473,"y":199.99998474121094,"wires":[["f9b9e089.404b3"]]},{"id":"8be76cd5.757438","type":"debug","z":"947cab1a.0d5bf","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","x":467,"y":374.5999755859375,"wires":}]
---
### Author:  [@Steve-Mcl](https://discourse.nodered.org/u/Steve-Mcl)
#### Post date: [5 August 2019 06:58 UTC](https://discourse.nodered.org/t/while-passing-array-to-node-and-reformating-array-into-an-object-data-gets-lost/14005/2 "2019-08-05T06:58:34Z")
Hi your code cannot be imported so I am not able to help.
Please put three back ticks above and below...
`````
`like this`
`````
> [@annibauer](#):
>
> When consoling the object only 1000 of the data points appear
The debug output only displays a portion (the real msg will have everything.
---
### Author:  [@annibauer](https://discourse.nodered.org/u/annibauer)
#### Post date: [5 August 2019 08:52 UTC](https://discourse.nodered.org/t/while-passing-array-to-node-and-reformating-array-into-an-object-data-gets-lost/14005/3 "2019-08-05T08:52:55Z")
```auto
[
{
"id": "947cab1a.0d5bf",
"type": "tab",
"label": "Flow 5",
"disabled": false,
"info": ""
},
{
"id": "65c95823.a10ae",
"type": "http request",
"z": "947cab1a.0d5bf",
"name": "",
"method": "POST",
"ret": "obj",
"paytoqs": true,
"url": "http://192.168.183.2:8800/ppm/v2/measurement/",
"tls": "",
"proxy": "",
"authType": "basic",
"x": 530,
"y": 300,
"wires": [
[
"f98141a.60f2a4"
]
]
},
{
"id": "f98141a.60f2a4",
"type": "debug",
"z": "947cab1a.0d5bf",
"name": "",
"active": false,
"tosidebar": true,
"console": true,
"tostatus": false,
"complete": "true",
"targetType": "full",
"x": 710,
"y": 300,
"wires": []
},
{
"id": "b6e0592a.23c17",
"type": "inject",
"z": "947cab1a.0d5bf",
"name": "",
"topic": "",
"payload": "",
"payloadType": "date",
"repeat": "",
"crontab": "",
"once": false,
"onceDelay": 0.1,
"x": 100,
"y": 120,
"wires": [
[
"55e126aa.826ec"
]
]
},
{
"id": "ba311656.326d98",
"type": "csv",
"z": "947cab1a.0d5bf",
"name": "",
"sep": ",",
"hdrin": true,
"hdrout": "",
"multi": "one",
"ret": "\\n",
"temp": "",
"skip": "0",
"x": 870,
"y": 120,
"wires": [
[
"82139ff5.5695f"
]
]
},
{
"id": "9b1fe86b.85791",
"type": "file in",
"z": "947cab1a.0d5bf",
"name": "",
"filename": "C:\\Users\\ABN1LR\\Desktop\\weldingDataFilter.csv",
"format": "utf8",
"chunk": false,
"sendError": false,
"encoding": "none",
"x": 590,
"y": 120,
"wires": [
[
"ba311656.326d98"
]
]
},
{
"id": "82139ff5.5695f",
"type": "function",
"z": "947cab1a.0d5bf",
"name": "Topic Added",
"func": "msg.topic = msg.payload.timerName;\nreturn msg;",
"outputs": 1,
"noerr": 0,
"x": 250,
"y": 200,
"wires": [
[
"c5e4f18c.4084e8"
]
]
},
{
"id": "f9b9e089.404b3",
"type": "function",
"z": "947cab1a.0d5bf",
"name": "Data Processing",
"func": "full_batch = msg.payload;\nspatter_running_total = 0;\n\nvar spatter_rate_history = flow.get('spatter_rate_history') || {};\nvar spatter_running_total = flow.get('spatter_rate_running_total') || {};\nvar spatter_count = flow.get('spatter_count') || {};\n\nvar body_array = []\n\nfor (var message_id in full_batch){\n \n if((full_batch[message_id][\"stabilisationFactorActValue\"] !== 0) ){\n\n timerName = full_batch[message_id][\"timerName\"];\n time = full_batch[message_id].dateTime;\n time_split = time.split(\"/\");\n time_split_time = time_split[2].split(\" \");\n time_split_hour_minute = time_split_time[1].split(\":\");\n time_split_hour = \"0\"+ time_split_hour_minute[0];\n hour = time_split_hour.substring(time_split_hour.length-2, time_split_hour.length+1);\n time_split_minute = \"0\"+ time_split_hour_minute[1];\n minute = time_split_minute.substring(time_split_minute.length-2, time_split_minute.length+1);\n var month = \"03\" //\"0\"+ time_split[0];\n month = month.substring(month.length-2, month.length+1);\n var day = \"0\"+ (parseInt(time_split[1])-15);\n day = day.substring(day.length-2, day.length+1);\n var year = \"2019\" //\"20\" + time_split_time[0];\n \n time_formatted = year + \"-\" + month + \"-\" + day + \"T\" + hour + \":\" + minute + \":00Z\";\n \n if (timerName in spatter_running_total){\n if (full_batch[message_id][\"uirExpulsionTime\"] !== 0){\n spatter_running_total[timerName] +=1;\n }\n }\n else{\n if (full_batch[message_id][\"uirExpulsionTime\"] !== 0){\n spatter_running_total[timerName] =1;\n }\n spatter_running_total[timerName] = 0;\n }\n \n if (timerName in spatter_count){\n spatter_count[timerName] +=1;\n }\n else{\n spatter_count[timerName] = 1;\n }\n \n body_array.push(\n {\n \"ts\": time_formatted,\n \"result\": \"OK\",\n \"series\": {\n \"$_time\": [\n 0\n],\n \"PSF\": [\n full_batch[message_id].stabilisationFactorActValue\n ],\n \"UIP\":[\n full_batch[message_id].uipActualValue\n ],\n \"Spatter Rate\": [\n spatter_running_total[timerName]/spatter_count[timerName]\n ],\n \"Voltage\": [\n full_batch[message_id].voltageActualValue\n ],\n \"Current\":[\n full_batch[message_id].currentActualValue\n ],\n \"uirExpulsionTime\":[\n full_batch[message_id].uirExpulsionTime\n ]\n }\n }\n \n );\n \n if (timerName in spatter_rate_history){\n spatter_rate_history[timerName].push(spatter_running_total[timerName]/spatter_count[timerName]);\n }\n else{\n spatter_rate_history[timerName] = []\n spatter_rate_history[timerName].push(spatter_running_total[timerName]/spatter_count[timerName]);\n }\n }\n \n}\n\nflow.set('spatter_rate_running_total', spatter_running_total);\nflow.set('spatter_count',spatter_count);\nflow.set('spatter_rate_history', spatter_rate_history);\n\nmsg.payload = body_array;\n\nreturn msg;\n\n\n",
"outputs": 1,
"noerr": 0,
"x": 720,
"y": 200,
"wires": [
[
"911a10da.a93a58"
]
]
},
{
"id": "911a10da.a93a58",
"type": "function",
"z": "947cab1a.0d5bf",
"name": "Request Body Formatting",
"func": "// MSG.PAYLOAD HOLDS THE DATA POINT ARRAY\n// when consoling the whole msg.payload from this node there is always a limit of 1000 data points\ncounter_test = msg.payload.length;\n \n msg.payload = {\n \"content-spec\": \"urn:spec://eclipse.org/unide/measurement-message#v2\",\n \"device\": {\n \"deviceID\": msg.topic\n },\n \"measurements\": msg.payload\n} ;\n\nmsg.headers = {'content-type':'application/json'};\nmsg.counter = counter_test;\nreturn msg;\n\n\n",
"outputs": 1,
"noerr": 0,
"x": 270,
"y": 300,
"wires": [
[
"8be76cd5.757438"
]
]
},
{
"id": "55e126aa.826ec",
"type": "function",
"z": "947cab1a.0d5bf",
"name": "Reset flow variables",
"func": "var spatter_rate_history = flow.get('spatter_rate_history') || {};\nvar spatter_running_total = flow.get('spatter_rate_running_total') || {};\nvar spatter_count = flow.get('spatter_count') || {};\n\nflow.set('spatter_rate_running_total', {});\nflow.set('spatter_count', {});\nflow.set('spatter_rate_history', {});\n\nreturn msg;",
"outputs": 1,
"noerr": 0,
"x": 280,
"y": 120,
"wires": [
[
"9b1fe86b.85791"
]
]
},
{
"id": "c5e4f18c.4084e8",
"type": "batcher",
"z": "947cab1a.0d5bf",
"name": "",
"maxTopics": "5",
"maxMessagesPerTopic": "500000",
"maxDelay": "1000",
"x": 473,
"y": 199.99998474121094,
"wires": [
[
"f9b9e089.404b3"
]
]
},
{
"id": "8be76cd5.757438",
"type": "debug",
"z": "947cab1a.0d5bf",
"name": "",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "false",
"x": 467,
"y": 374.5999755859375,
"wires": []
}
]
```
---
### Author:  [@TotallyInformation](https://discourse.nodered.org/u/TotallyInformation)
#### Post date: [5 August 2019 12:16 UTC](https://discourse.nodered.org/t/while-passing-array-to-node-and-reformating-array-into-an-object-data-gets-lost/14005/4 "2019-08-05T12:16:40Z")
Are you checking the ouput using a debug node? If so, that is limited to about 1k characters by default.