Splitting a Serial string in to individual packets

General
1

Good evening

Hoping someone can point me in the right direction, I need to remove individual packets (so i can further extract the info i need) from a long serial string of several packets, that also seem to change order.

I've used the Serial node and am getting a string of Decimal showing in NodeRED.

I have the proctocol documentation and the data looks correct, Initally I think I just need to break it in to the smaller packets, and filter out the ones I'm interested to then convert the Decimal bytes to Ascii and extract and process the parts I need to pass on.

Here's an example of the String received from the serial:-

[129,32,50,50,58,48,48,48,48,32,32,63,130,49,49,48,32,50,57,32,52,32,32,51,131,32,32,32,32,32,32,32,50,32,50,103,148,32,51,32,32,51,32,32,32,32,32,122,132,32,32,32,32,32,32,32,32,32,32,68,133,32,32,32,32,32,32,32,32,32,32,69,134,32,32,32,32,32,32,32,32,32,32,70,135,32,32,32,32,32,32,32,32,32,32,71,136,32,32,32,32,32,32,32,32,32,32,72,137,32,32,32,32,32,32,32,32,32,32,73,138,32,32,32,32,32,32,32,32,32,32,74,139,32,32,32,32,32,32,32,32,32,32,75,140,32,32,32,32,32,32,32,32,32,32,76,141,32,32,32,32,32,32,32,32,32,32,77,142,32,32,32,32,32,32,32,32,32,32,78,143,32,32,32,32,32,32,32,32,32,32,79,144,32,32,32,32,32,32,32,32,32,32,80,145,32,32,32,32,32,32,32,32,32,32,81,192,32,32,32,32,32,32,32,80,32,32,48,193,32,32,32,32,32,32,32,80,32,32,49,194,32,32,32,32,32,32,32,80,32,32,50,195,32,32,32,32,32,32,32,80,32,32,51,196,32,32,32,32,32,32,32,80,32,32,52,197,32,32,32,32,32,32,32,80,32,32,53,198,32,32,32,32,32,32,32,80,32,32,54,199,32,32,32,32,32,32,32,80,32,32,55,200,32,32,32,32,32,32,32,80,32,32,56,201,32,32,32,32,32,32,32,80,32,32,57,202,32,32,32,32,32,32,32,80,32,32,58,203,32,32,32,32,32,32,32,80,32,32,59,204,32,32,32,32,32,32,32,80,32,32,60,205,32,32,32,32,32,32,32,80,32,32,61,206,32,32,32,32,32,32,32,80,32,32,62,208,32,32,32,32,32,32,32,80,32,32,64,209,32,32,32,32,32,32,32,80,32,32,65,210,32,32,32,32,32,32,32,80,32,32,66,211,32,32,32,32,32,32,32,80,32,32,67,212,32,32,32,32,32,32,32,80,32,32,68,213,32,32,32,32,32,32,32,80,32,32,69,214,32,32,32,32,32,32,32,80,32,32,70,215,32,32,32,32,32,32,32,80,32,32,71,216,32,32,32,32,32,32,32]

Each packet starts with a 3 digit number (address) and is 12 or 14 bytes long, is there a way I could break these up in to individual payloads?
I could then filter on the first byte (the address) as to wether or not I the info in that packet, and then convert the needed packets to ascii to reveal the info to then pass on to the next stage.

Kind Regards

2

This is a string, correct? Complete with quotes and everything as it is listed in a debug node or wherever you're pulling it from? If so, there's a few ways you could do this. The first way I can think of (and the way I would chose to do this) goes like this.

image
image972×364 17.3 KB

Details:

[{"id":"8e5acf8c4188841a","type":"inject","z":"4761ebd8ef29acca","name":"","props":[{"p":"payload"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[129,32,50,50,58,48,48,48,48,32,32,63,130,49,49,48,32,50,57,32,52,32,32,51,131,32,32,32,32,32,32,32,50,32,50,103,148,32,51,32,32,51,32,32,32,32,32,122,132,32,32,32,32,32,32,32,32,32,32,68,133,32,32,32,32,32,32,32,32,32,32,69,134,32,32,32,32,32,32,32,32,32,32,70,135,32,32,32,32,32,32,32,32,32,32,71,136,32,32,32,32,32,32,32,32,32,32,72,137,32,32,32,32,32,32,32,32,32,32,73,138,32,32,32,32,32,32,32,32,32,32,74,139,32,32,32,32,32,32,32,32,32,32,75,140,32,32,32,32,32,32,32,32,32,32,76,141,32,32,32,32,32,32,32,32,32,32,77,142,32,32,32,32,32,32,32,32,32,32,78,143,32,32,32,32,32,32,32,32,32,32,79,144,32,32,32,32,32,32,32,32,32,32,80,145,32,32,32,32,32,32,32,32,32,32,81,192,32,32,32,32,32,32,32,80,32,32,48,193,32,32,32,32,32,32,32,80,32,32,49,194,32,32,32,32,32,32,32,80,32,32,50,195,32,32,32,32,32,32,32,80,32,32,51,196,32,32,32,32,32,32,32,80,32,32,52,197,32,32,32,32,32,32,32,80,32,32,53,198,32,32,32,32,32,32,32,80,32,32,54,199,32,32,32,32,32,32,32,80,32,32,55,200,32,32,32,32,32,32,32,80,32,32,56,201,32,32,32,32,32,32,32,80,32,32,57,202,32,32,32,32,32,32,32,80,32,32,58,203,32,32,32,32,32,32,32,80,32,32,59,204,32,32,32,32,32,32,32,80,32,32,60,205,32,32,32,32,32,32,32,80,32,32,61,206,32,32,32,32,32,32,32,80,32,32,62,208,32,32,32,32,32,32,32,80,32,32,64,209,32,32,32,32,32,32,32,80,32,32,65,210,32,32,32,32,32,32,32,80,32,32,66,211,32,32,32,32,32,32,32,80,32,32,67,212,32,32,32,32,32,32,32,80,32,32,68,213,32,32,32,32,32,32,32,80,32,32,69,214,32,32,32,32,32,32,32,80,32,32,70,215,32,32,32,32,32,32,32,80,32,32,71,216,32,32,32,32,32,32,32]","payloadType":"str","x":870,"y":120,"wires":[["846bc3d68800a970"]]},{"id":"846bc3d68800a970","type":"json","z":"4761ebd8ef29acca","name":"","property":"payload","action":"","pretty":false,"x":990,"y":120,"wires":[["e27d1c935bd5ff73","bb8d06ac16cbcd20"]]},{"id":"e27d1c935bd5ff73","type":"debug","z":"4761ebd8ef29acca","name":"debug 1","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1060,"y":200,"wires":[]},{"id":"bb8d06ac16cbcd20","type":"function","z":"4761ebd8ef29acca","name":"function 1","func":"var stringArr = msg.payload;\nvar bigArr = {};\nvar loopArr;\nvar shiftIn;\nwhile(stringArr.length > 0){\n    loopArr = [];\n    shiftIn = stringArr.shift();\n    do{\n        loopArr.push(stringArr.shift());\n    }while(stringArr[0] < 100);\n    bigArr[shiftIn] = loopArr;\n}\nmsg.payload = bigArr;\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":1120,"y":120,"wires":[["3e895405226ba3d8"]]},{"id":"3e895405226ba3d8","type":"debug","z":"4761ebd8ef29acca","name":"debug 2","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1260,"y":120,"wires":[]}]

It's basic. It's probably poor coding. But it works and gives you keyed pairs to all the values that follow. That way you can reference what I'm assuming is a specific register or whatever and pull some kind of historical data from it. Just substitute whatever you're getting your string from for the inject node and debug 2 gets whatever uses the keyed pairs and Bob's your uncle.

In any case, that's the method I would use.

3

It would be useful to see that otherwise we have to guess what the start/end markers are, whether there is a checksum etc etc etc

4

Hi Thanks for input so far, its probably my miss understanding of the message type that is been outputted from the serial.

Here's the serial to the debug

image
image1116×589 27.6 KB

image
image421×733 13.1 KB

They layout when opened up looks to be an Array, but the sting of Decimal does tie to the documentation i have. (I'll post seprately about the protocol).

5

Is this getting a periodic deposit of an array or is it a continuous stream? What you have so far looks like the serial gets a burst of data and then goes idle (hence the independent messages instead of a raw stream). If it's a raw and continuous stream, my solution above would still work, but you would need to read the stream one array component at a time and determine if it's a key or a value and store accordingly. If it's a burst like it looks like, the solution above will still take care of it.

But I'll wait until I see the protocol before I adjust anything.

6

image
image1419×659 115 KB

For example the begining of every 2nd string starts 129, ...... and looking at the following bytes are correct with the information I expected, from the console.

image
image1081×556 66.6 KB

but there's nothing special about the end of the packet, the next packet follows straight on.

Sure there's a better way, but in my simple approach I was going to replace each address with a topic, ie, Replace '129,' with ' : ShotClock,' & '128,' with ' : Game Clock,' the in the next step I could split the whole payload string based on the ':' and end up with smaller packets - removing ones i dont need and processing ones I do need info from.

Happy to hear how i should be doing it properly though.

7

230-S11-EN - FS1 Protocol.pdf (345.3 KB)

Here's the full document if it helps.

8

It seems to have regular breaks as first string seems to allways strat 129, .... 2nd always 173, .... but it repeats containusly (its a controller to a score board for basket ball).

I did put the serial straight in to your code to see what I get, but just get an error - thin thats because I've assumed the wrong payload type?

image

Wasn't sure if the JSon Node was to help you mimic the data string i have or a step in the process, but have an error with both.

below missing out the JSon Node

image

9

Ok. Your first error is a syntax error. It looks like the JSON node is receiving something it's not expecting. I set it up to respond to a string. If you already have an array coming out, you don't need the JSON node. It was purely to parse an actual string into an array and can be removed if you have an array.

The TypeError is possibly the same thing. .shift() is a module of array type variables that pops the first element in an array off the array and into the receiving object. If you don't have an array, you don't have the .shift() module associated with it. You have to figure out what data type you have coming out of your serial (array, string, object, something else...) and respond accordingly.

Out of curiosity, why are you pulling a serial output of a scoreboard controller? What is Node-Red doing for you with it? It sounds like you're tapping into the serial communications between the controller and the board itself, based on your description and the repetitive codes. The trick with something like that is the controller will send the same sequence of registers all the time and vary the data to cause the board to change it's display and such. If you know the sequence, it will be much easier just to setup a parse of each packet based on known parameters than trying to parse it based on just receiving something that seems unstructured. The packets will likely be the same length and order and repeat in a given sequence. Better if you know that and can work off of it because you can not only parse your data better, but you could possibly take the values and turn them into something more meaningful than just an array.

But again, go back to the beginning. Figure out what you have coming out of the serial node and adjust accordingly. That will remove your JSON error and possibly eliminate your .shift() syntax error.

10

If all you require is the address separated and the converted to ascii then this should work. You can the decode further using msg.topic to route the address.
That is if understand correctly.

[{"id":"c7fec782ec40883e","type":"inject","z":"366a43adb328cf95","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[129,32,50,50,58,48,48,48,48,32,32,63,130,49,49,48,32,50,57,32,52,32,32,51,131,32,32,32,32,32,32,32,50,32,50,103,148,32,51,32,32,51,32,32,32,32,32,122,132,32,32,32,32,32,32,32,32,32,32,68,133,32,32,32,32,32,32,32,32,32,32,69,134,32,32,32,32,32,32,32,32,32,32,70,135,32,32,32,32,32,32,32,32,32,32,71,136,32,32,32,32,32,32,32,32,32,32,72,137,32,32,32,32,32,32,32,32,32,32,73,138,32,32,32,32,32,32,32,32,32,32,74,139,32,32,32,32,32,32,32,32,32,32,75,140,32,32,32,32,32,32,32,32,32,32,76,141,32,32,32,32,32,32,32,32,32,32,77,142,32,32,32,32,32,32,32,32,32,32,78,143,32,32,32,32,32,32,32,32,32,32,79,144,32,32,32,32,32,32,32,32,32,32,80,145,32,32,32,32,32,32,32,32,32,32,81,192,32,32,32,32,32,32,32,80,32,32,48,193,32,32,32,32,32,32,32,80,32,32,49,194,32,32,32,32,32,32,32,80,32,32,50,195,32,32,32,32,32,32,32,80,32,32,51,196,32,32,32,32,32,32,32,80,32,32,52,197,32,32,32,32,32,32,32,80,32,32,53,198,32,32,32,32,32,32,32,80,32,32,54,199,32,32,32,32,32,32,32,80,32,32,55,200,32,32,32,32,32,32,32,80,32,32,56,201,32,32,32,32,32,32,32,80,32,32,57,202,32,32,32,32,32,32,32,80,32,32,58,203,32,32,32,32,32,32,32,80,32,32,59,204,32,32,32,32,32,32,32,80,32,32,60,205,32,32,32,32,32,32,32,80,32,32,61,206,32,32,32,32,32,32,32,80,32,32,62,208,32,32,32,32,32,32,32,80,32,32,64,209,32,32,32,32,32,32,32,80,32,32,65,210,32,32,32,32,32,32,32,80,32,32,66,211,32,32,32,32,32,32,32,80,32,32,67,212,32,32,32,32,32,32,32,80,32,32,68,213,32,32,32,32,32,32,32,80,32,32,69,214,32,32,32,32,32,32,32,80,32,32,70,215,32,32,32,32,32,32,32,80,32,32,71,216,32,32,32,32,32,32,32]","payloadType":"str","x":170,"y":200,"wires":[["8b0f52dbc8e6a377"]]},{"id":"8b0f52dbc8e6a377","type":"json","z":"366a43adb328cf95","name":"","property":"payload","action":"","pretty":false,"x":290,"y":200,"wires":[["93a59e9266169958"]]},{"id":"93a59e9266169958","type":"split","z":"366a43adb328cf95","name":"","splt":"\\n","spltType":"str","arraySplt":"12","arraySpltType":"len","stream":false,"addname":"","x":410,"y":200,"wires":[["9ccbde1a5d88a029","56a12b0b04cf3e04"]]},{"id":"9ccbde1a5d88a029","type":"function","z":"366a43adb328cf95","name":"function 12","func":"msg.topic = msg.payload[0];\nmsg.payload=Buffer.from(msg.payload.slice(1,-1)).toString()\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":550,"y":200,"wires":[["031b1d1081de2795"]]},{"id":"56a12b0b04cf3e04","type":"debug","z":"366a43adb328cf95","name":"debug 91","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":680,"y":140,"wires":[]},{"id":"031b1d1081de2795","type":"join","z":"366a43adb328cf95","name":"","mode":"reduce","build":"object","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"","count":"","reduceRight":false,"reduceExp":"$merge([$A,{$string($$.topic):$$.payload}])","reduceInit":"{}","reduceInitType":"json","reduceFixup":"","x":610,"y":260,"wires":[["1a6d4ce2cf39d27a"]]},{"id":"1a6d4ce2cf39d27a","type":"debug","z":"366a43adb328cf95","name":"debug 93","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":620,"y":300,"wires":[]}]

Output

chrome_screenshot_1665606625270
chrome_screenshot_16656066252701462×2048 98.5 KB

chrome_screenshot_1665606693475
chrome_screenshot_16656066934751493×2048 95.6 KB

11

Yes, thanks for putting that up there. That was going to be my next point. Once the different register patterns are known, we can pull them out and manipulate them as needed. It may be that some are ascii representations such as the time above that is rather obvious in your debug, or they may be some other representation such as a command or whatever.

@DAWaller79, you'll want to really dig into your system to figure out what it's putting out and what you need to do with it. As E1cid pointed out, you can do a lot with your data once you know what it is and what you're getting. But you have to know what you're receiving first so you can manipulate it correctly to make it usable to you. Datasheets may be your best friend at this point.

12

Hi Both @madhouse @E1cid

I've Played with both your Flows and they seem to be on the right path, (hoping I understand them properly), think i have two issues at present.

1st the payload from the serial node isnt as expected, the debug says its a buffer? I tested other ways that manipulate string and arrays gave me errors in earlier experiments - guessing i'm not converting it right? any pointers.

2nd the packets can be 12 or 14 bytes long, and i think can change order if something updates on the controller, so in E1cid example you split evert 12? this will give erronous data after a 14 byte packet?

13

That is common for many Node-Red nodes that communicate with outside peripheral devices, such as serial, MODBUS, COM data or anything that would be "unstructured". This is simply because the node handles the communication and passes in the raw data. Since it doesn't know what the data is, it passes it as a buffer, which is another name for a raw data packet. When you get something like msg.payload, the data is known, structured and capable of being manipulated by assigned modules. With buffers, the data is unknown and therefore unstructured and not capable of inherent manipulations.

This is all part of knowing what's coming in. Once you know what's being passed, you know what to do with it. If your data is an address with 12-14 8bit characters associated to it, then you need a function node where you work through the buffer and turn it into structured data. If some are characters and others are floats or doubles, you'll need to know that. If some are commands, you'll need to know that. That's what both myself and E1cid were trying to show is that data conditioning. My function pulled out the raw buffer array and put it into an address and associated data array where E1cid showed how to convert the data to something readable and usable in future nodes. Once you know what you're receiving, you can figure out how to parse it and how to condition it to something usable. If you combine what we've been showing you and add some icing to the cake, you can have exactly what you're looking for. But that's going to require a little coding on your end to make sure what's coming in gets separated and converted into what you need so you can use it properly.

14

Hi @madhouse

Ah, ok hadn't considered that, expected the serial node to bring in a string or something.

I have just been trying converting a buffer to string with reasonalable sucess. However seem to note the output is an ascii payload, first byte (the address - I'll use to identify each type of information i need) of each packet, is convered to an ? in a diamond and when converted back to decimal all addresses are 6553 (other bytes are ok) I guess thats because the address is not reconised as an ascii character? therefore I'll need to process it in decimal first?

Did you spot the documentation I posted earlier? Looking at the individual packets the only way I can think to split it is to use the address, it will only ever appear once in each burst, and can be used to identify the packets. Just need to make sure i can manipulate the payload.

Thankyou for your help so far.

15

This may help

[{"id":"c7fec782ec40883e","type":"inject","z":"366a43adb328cf95","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[129,32,50,50,58,48,48,48,48,32,32,63,130,49,49,48,32,50,57,32,52,32,32,51,131,32,32,32,32,32,32,32,50,32,50,103,148,32,51,32,32,51,32,32,32,32,32,122,132,32,32,32,32,32,32,32,32,32,32,68,133,32,32,32,32,32,32,32,32,32,32,69,134,32,32,32,32,32,32,32,32,32,32,70,135,32,32,32,32,32,32,32,32,32,32,71,136,32,32,32,32,32,32,32,32,32,32,72,137,32,32,32,32,32,32,32,32,32,32,73,138,32,32,32,32,32,32,32,32,32,32,74,139,32,32,32,32,32,32,32,32,32,32,75,140,32,32,32,32,32,32,32,32,32,32,76,141,32,32,32,32,32,32,32,32,32,32,77,142,32,32,32,32,32,32,32,32,32,32,78,143,32,32,32,32,32,32,32,32,32,32,79,144,32,32,32,32,32,32,32,32,32,32,80,145,32,32,32,32,32,32,32,32,32,32,81,192,32,32,32,32,32,32,32,80,32,32,48,193,32,32,32,32,32,32,32,80,32,32,49,194,32,32,32,32,32,32,32,80,32,32,50,195,32,32,32,32,32,32,32,80,32,32,51,196,32,32,32,32,32,32,32,80,32,32,52,197,32,32,32,32,32,32,32,80,32,32,53,198,32,32,32,32,32,32,32,80,32,32,54,199,32,32,32,32,32,32,32,80,32,32,55,200,32,32,32,32,32,32,32,80,32,32,56,201,32,32,32,32,32,32,32,80,32,32,57,202,32,32,32,32,32,32,32,80,32,32,58,203,32,32,32,32,32,32,32,80,32,32,59,204,32,32,32,32,32,32,32,80,32,32,60,205,32,32,32,32,32,32,32,80,32,32,61,206,32,32,32,32,32,32,32,80,32,32,62,208,32,32,32,32,32,32,32,80,32,32,64,209,32,32,32,32,32,32,32,80,32,32,65,210,32,32,32,32,32,32,32,80,32,32,66,211,32,32,32,32,32,32,32,80,32,32,67,212,32,32,32,32,32,32,32,80,32,32,68,213,32,32,32,32,32,32,32,80,32,32,69,214,32,32,32,32,32,32,32,80,32,32,70,215,32,32,32,32,32,32,32,80,32,32,71,216,32,32,32,32,32,32,32]","payloadType":"bin","x":130,"y":240,"wires":[["42ad02f6f497c370"]]},{"id":"42ad02f6f497c370","type":"function","z":"366a43adb328cf95","name":"function 13","func":"const config = {send: 1, output:1}; // 1 / 0 to node.send() / output object\nlet output = {};\nlet key = msg.payload[0]\nlet arr = []\nmsg.payload.slice(1).forEach(val => {\n    if(Number(val) >= 128){\n        output[key] = Buffer.from(arr.slice(0,-1)).toString()\n        if(config.send) node.send({topic: key, payload: output[key]});\n        key = val;\n        arr = [];\n    }else{\n       arr.push(val)\n    }\n})\n// add and send final line if needed\nif(arr.length > 0){\n    output[key] = Buffer.from(arr.slice(0,-1)).toString();\n    if(config.send) node.send({topic: key, payload: output[key]});\n}\nif(config.output) msg.payload = output;\nelse msg = null;\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":270,"y":240,"wires":[["e27d1c935bd5ff73"]]},{"id":"e27d1c935bd5ff73","type":"debug","z":"366a43adb328cf95","name":"debug 1","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":460,"y":140,"wires":[]}]
16

Hi @E1cid

Perfect, now just to work out how its done! (for my own curiosity) that should pass perfectly in to how I planned on procceding.
I've not picked though it yet, and haven't spotted any but is that handling both the 12 & 14 byte packets?

Thankyou for your help.

17

It should handle any length message it looks for address above 127 and then decodes the ascii and then starts a new output property for next data set.

18

Once i'd looked at the code i thought thats what it did. thats a great help.

Thankyou all.

19

Welcome. Glad we could help. Good learning experience with external peripherals!

20

This topic was automatically closed 60 days after the last reply. New replies are no longer allowed.