BestPractice MQTT topic/# into JSON File

General
,
1

Hi,
I am quite a newbee to Node-Red, but I am starting to love it :-).
I know some JS and I managed to grab my sensordata via MQTT and other data offtopic ways (from homematic via http request) and further I do fill an influxdb with Node-Red.

But I am still missing the big picture here:
(and I really tried hard for days to follow and search the forum |-))

What would be the best practice when I grab my sensordata from sensor/# to build a single JSON File for another remoteplattform with ALL the data that comes within that topic? And how todo that.

So simplified I want to accomplish something like this:

Data via MQTT sensor/status, sensor/value1, sensor/value2
form into a JSON like this:
{ "name" : "sensor", "status":"online", "value1":100, "value2":20 }

I allready managed to write a predefined JSON to file, so I am really just missing
how to take msg.top and msg.payload dynamicaly and add it to a global var that can than be written to file.

Any hint which leads me further to my target is appreciated

best regards
A. B.

2

So you have an mqtt-in node that listens on sensor/# and therefore recieves all the data. The easiest approach (for me anyway) is to put in a function node that references a context variable. That variable can use the node's context if you plan to add all current data to an output message each time you recieve a sensor update. Or it can use a flow or global context if you have a separate part of your flows that needs the data.

// Get the context var (a node context here) or create a blank object if the var doesn't exist
const sensors = context.get('sensors') || {}

const splitTopic = msg.topic.split('/')

sensors[ splitTopic[0] ][ splitTopic[1] ] = msg.payload

context.set('sensors', sensors)

return { 'payload': sensors }

What I would say is that if you have multiple sensors, you probably want to chose an encompasing top-level topic - say sensors/sensor1/temperature etc. Then you can listen for and process ALL sensor data with a single flow.

3

Hi @daspixel, welcome to the forum.

What is this "other remote platform"? Can it subscribe to the same MQTT broker as Node-red?
What exactly do you mean by "ALL the data"?
Can you explain why you want to put this data into a file, especially since you already have it in Influxdb?

4

so you know (I hope) that using the # in the mqtt topic gets every thing published using 'sensor' so I'll assume you are getting more that one msg that you want to combine.

NR process one msg at a time so you need to save the data or join multiple msgs before you can create another msg with the data combined.

Since it seems like you are receiving three msgs, you could add a switch node after and a change node after to set the topic then feed them into a join like this:

Screen Shot 2022-10-25 at 6.57.10 AM
Screen Shot 2022-10-25 at 6.57.10 AM960Ă—292 30.7 KB

here is the flow:

[{"id":"a079c2c6d85e8f14","type":"tab","label":"Flow 8","disabled":false,"info":"","env":[]},{"id":"af82cfa41f20293b","type":"group","z":"a079c2c6d85e8f14","name":"","style":{"fill":"#dbcbe7","label":true},"nodes":["98d8829914d9517d","ef899ba32e2d594c","33b7885297a32331","ecf9d463a1def8d2"],"x":34,"y":19,"w":292,"h":262},{"id":"98d8829914d9517d","type":"inject","z":"a079c2c6d85e8f14","g":"af82cfa41f20293b","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"sensor/status","payload":"online","payloadType":"str","x":190,"y":120,"wires":[["44d59c1e8b762e11"]]},{"id":"ef899ba32e2d594c","type":"inject","z":"a079c2c6d85e8f14","g":"af82cfa41f20293b","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"sensor/value1","payload":"100","payloadType":"num","x":190,"y":180,"wires":[["44d59c1e8b762e11"]]},{"id":"33b7885297a32331","type":"inject","z":"a079c2c6d85e8f14","g":"af82cfa41f20293b","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"sensor/value2","payload":"20","payloadType":"num","x":180,"y":240,"wires":[["44d59c1e8b762e11"]]},{"id":"44d59c1e8b762e11","type":"switch","z":"a079c2c6d85e8f14","name":"","property":"topic","propertyType":"msg","rules":[{"t":"eq","v":"sensor/status","vt":"str"},{"t":"eq","v":"sensor/value1","vt":"str"},{"t":"eq","v":"sensor/value2","vt":"str"}],"checkall":"true","repair":false,"outputs":3,"x":390,"y":180,"wires":[["a84d6c898ca30f2a"],["e20a825e4ba2fd18"],["78e2769a789720cb"]]},{"id":"724ff8d8f7706e87","type":"debug","z":"a079c2c6d85e8f14","name":"debug 29","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":860,"y":180,"wires":[]},{"id":"f2be277702a276d0","type":"join","z":"a079c2c6d85e8f14","name":"","mode":"custom","build":"object","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":true,"timeout":"","count":"3","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":710,"y":180,"wires":[["724ff8d8f7706e87"]]},{"id":"a84d6c898ca30f2a","type":"change","z":"a079c2c6d85e8f14","name":"status","rules":[{"t":"set","p":"topic","pt":"msg","to":"status","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":550,"y":140,"wires":[["f2be277702a276d0"]]},{"id":"e20a825e4ba2fd18","type":"change","z":"a079c2c6d85e8f14","name":"value1","rules":[{"t":"set","p":"topic","pt":"msg","to":"value1","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":550,"y":180,"wires":[["f2be277702a276d0"]]},{"id":"78e2769a789720cb","type":"change","z":"a079c2c6d85e8f14","name":"value2","rules":[{"t":"set","p":"topic","pt":"msg","to":"value2","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":550,"y":220,"wires":[["f2be277702a276d0"]]},{"id":"ecf9d463a1def8d2","type":"comment","z":"a079c2c6d85e8f14","g":"af82cfa41f20293b","name":"emulating one mqtt-in node","info":"","x":180,"y":60,"wires":[]}]
5

Hi,
thank you very much :star_struck: for the explaination and the code - I will try it instant :slight_smile: - I guess I need some days for a full implementation, but I will give more feedback as soon as I have succeeded.
I will just start a small testsession :smiley:
A. B.

6

Hi,
thank you for taking time to read, the remote plattform and my "sensor"-platform are not connected over the Net, there is no chance to tunnel or do anything fancy, my only choice is to do a rsync over 2 hops.
Second I love json for storing data in a very easy way, because I have lots of possibilities to work with - even all those are not best ways - JSON just needs a small HTML and JS File to be displayed - no server, no dynamic things, works on the weakest device and it is also some kind of backup (no seroius one).
best regards
A. B.

7

Hi,
thank you also very much for your example and explaination, I thought about something like that, and as the example from "TotallyInformation" I will try it. I would prefer to do it with the visible way, but in real life I have about 34 sub-topics, so a programatic approach would be sleeker, but I like the wysiwig of NR.
Your example will help me to understand NR a little better - I will test and see how it reacts.
I will keep you updated.
best regards
A.B.

1 Like
8

It is possible to mix and match as you prefer. Personally I like node-red to do the hard work so that I can focus on small pieces of logic which function nodes are great for. But of course, I don't always use function nodes if the logic is easily described by other nodes.

But I think that here, the idea of being able to process ALL of your sensors in a single, simple flow is tremendously powerful and takes little coding. Doubtless there are ways to do the logic I described in nodes but it was easier for me (since I am reasonably OK at coding javascript) to quickly describe the logic in code.

Here is an example of using a mix of nodes and a central core of function nodes to handle many different sensor inputs, rationalise them and output to standard mqtt topics.

image
image1330Ă—621 101 KB

Whereas here is an example of processing a small number of related inputs using just nodes (controlling some lighting in my office):

image
image842Ă—464 32.9 KB

9

Wow, I really gratefully see the examples and read all the info from you guys!

As I do most of my programming late and at night - I very often have typos or logical bugs.

Tonight I managed to get a first small version of my Flow running with 5 subtopics the graphical way - after removing a wrong setting from a node (which came from copy/paste)!

Special Thanks to @zenofmud

I will keep this for tonight, but will try to code the function at weekend - the graphical nodes are huge with 34 topics :laughing:

best regards A. B.

10

Hi,
I just wanted to let you know, that I managed to do most of my wanted "actions"!
Thank you very much!

[
    {
        "id": "5ec0b74d01a77dd8",
        "type": "mqtt in",
        "z": "da74aa24b406fbc1",
        "name": "BKW DTU ALLE",
        "topic": "bkw/dtu/#",
        "qos": "0",
        "datatype": "auto-detect",
        "broker": "db6aefa0851d099a",
        "nl": false,
        "rap": false,
        "rh": 0,
        "inputs": 0,
        "x": 140,
        "y": 580,
        "wires": [
            [
                "b40b67f22656bf77",
                "a7296ea9a7ba391c"
            ]
        ]
    },
    {
        "id": "b40b67f22656bf77",
        "type": "switch",
        "z": "da74aa24b406fbc1",
        "name": "",
        "property": "topic",
        "propertyType": "msg",
        "rules": [
            {
                "t": "eq",
                "v": "bkw/dtu/ip",
                "vt": "str"
            },
            {
                "t": "eq",
                "v": "bkw/dtu/hostname",
                "vt": "str"
            },
            {
                "t": "eq",
                "v": "bkw/dtu/rssi",
                "vt": "str"
            },
            {
                "t": "eq",
                "v": "bkw/dtu/status",
                "vt": "str"
            },
            {
                "t": "eq",
                "v": "bkw/dtu/uptime",
                "vt": "str"
            }
        ],
        "checkall": "true",
        "repair": false,
        "outputs": 5,
        "x": 370,
        "y": 580,
        "wires": [
            [
                "7a5b68dbe1b8c06f",
                "7e6712ba8219810f"
            ],
            [
                "df2bb614e77dd845",
                "7e6712ba8219810f"
            ],
            [
                "d4199abcd9bb9528",
                "7e6712ba8219810f"
            ],
            [
                "f1f84751f7c1540d",
                "7e6712ba8219810f"
            ],
            [
                "148c2e510cd68245",
                "7e6712ba8219810f"
            ]
        ]
    },
    {
        "id": "7a5b68dbe1b8c06f",
        "type": "change",
        "z": "da74aa24b406fbc1",
        "name": "ip",
        "rules": [
            {
                "t": "set",
                "p": "topic",
                "pt": "msg",
                "to": "ip",
                "tot": "str"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 610,
        "y": 500,
        "wires": [
            [
                "ba1b631d46011df3",
                "19282e7f290e0f87"
            ]
        ]
    },
    {
        "id": "df2bb614e77dd845",
        "type": "change",
        "z": "da74aa24b406fbc1",
        "name": "hostname",
        "rules": [
            {
                "t": "set",
                "p": "topic",
                "pt": "msg",
                "to": "hostname",
                "tot": "str"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 620,
        "y": 540,
        "wires": [
            [
                "ba1b631d46011df3",
                "b41a2d6af5cb2e52"
            ]
        ]
    },
    {
        "id": "d4199abcd9bb9528",
        "type": "change",
        "z": "da74aa24b406fbc1",
        "name": "RSSI",
        "rules": [
            {
                "t": "set",
                "p": "topic",
                "pt": "msg",
                "to": "rssi",
                "tot": "str"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 610,
        "y": 640,
        "wires": [
            [
                "ba1b631d46011df3",
                "3ad39f65f46f252a"
            ]
        ]
    },
    {
        "id": "f1f84751f7c1540d",
        "type": "change",
        "z": "da74aa24b406fbc1",
        "name": "status",
        "rules": [
            {
                "t": "set",
                "p": "topic",
                "pt": "msg",
                "to": "status",
                "tot": "str"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 610,
        "y": 580,
        "wires": [
            [
                "ba1b631d46011df3",
                "3cadef2b36f2b9ab"
            ]
        ]
    },
    {
        "id": "148c2e510cd68245",
        "type": "change",
        "z": "da74aa24b406fbc1",
        "name": "uptime",
        "rules": [
            {
                "t": "set",
                "p": "topic",
                "pt": "msg",
                "to": "uptime",
                "tot": "str"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 610,
        "y": 680,
        "wires": [
            [
                "ba1b631d46011df3",
                "0b3e01c369b4472c"
            ]
        ]
    },
    {
        "id": "52a5eba435b1e589",
        "type": "join",
        "z": "da74aa24b406fbc1",
        "name": "",
        "mode": "custom",
        "build": "object",
        "property": "payload",
        "propertyType": "msg",
        "key": "topic",
        "joiner": "\\n",
        "joinerType": "str",
        "accumulate": true,
        "timeout": "",
        "count": "6",
        "reduceRight": false,
        "reduceExp": "",
        "reduceInit": "",
        "reduceInitType": "",
        "reduceFixup": "",
        "x": 1210,
        "y": 580,
        "wires": [
            [
                "211cb5852700edad",
                "a454653198085fdf"
            ]
        ]
    },
    {
        "id": "211cb5852700edad",
        "type": "debug",
        "z": "da74aa24b406fbc1",
        "name": "BKW Debug DTU",
        "active": false,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "payload",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 1430,
        "y": 520,
        "wires": []
    },
    {
        "id": "a7296ea9a7ba391c",
        "type": "debug",
        "z": "da74aa24b406fbc1",
        "name": "BKW DTU Debug MQTT",
        "active": false,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "payload",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 310,
        "y": 760,
        "wires": []
    },
    {
        "id": "7e6712ba8219810f",
        "type": "debug",
        "z": "da74aa24b406fbc1",
        "name": "BKW DTU SWITCH Debug",
        "active": false,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "payload",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 600,
        "y": 760,
        "wires": []
    },
    {
        "id": "ba1b631d46011df3",
        "type": "debug",
        "z": "da74aa24b406fbc1",
        "name": "BKW DTU TOPIC Debug",
        "active": false,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "payload",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 910,
        "y": 760,
        "wires": []
    },
    {
        "id": "a454653198085fdf",
        "type": "file",
        "z": "da74aa24b406fbc1",
        "name": "energie-pv-dtu.json",
        "filename": "/home/antoniobeemax/web-data/energie/data/energie-pv-dtu.json",
        "filenameType": "str",
        "appendNewline": true,
        "createDir": false,
        "overwriteFile": "true",
        "encoding": "utf8",
        "x": 1630,
        "y": 580,
        "wires": [
            []
        ]
    },
    {
        "id": "a1b650a2eff912f3",
        "type": "debug",
        "z": "da74aa24b406fbc1",
        "name": "BKW Debug DTU JSON",
        "active": false,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "payload",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 1650,
        "y": 640,
        "wires": []
    },
    {
        "id": "0b3e01c369b4472c",
        "type": "delay",
        "z": "da74aa24b406fbc1",
        "name": "Maximal alle 15min",
        "pauseType": "timed",
        "timeout": "5",
        "timeoutUnits": "seconds",
        "rate": "1",
        "nbRateUnits": "15",
        "rateUnits": "minute",
        "randomFirst": "1",
        "randomLast": "5",
        "randomUnits": "seconds",
        "drop": true,
        "allowrate": false,
        "outputs": 1,
        "x": 930,
        "y": 680,
        "wires": [
            [
                "52a5eba435b1e589"
            ]
        ]
    },
    {
        "id": "3ad39f65f46f252a",
        "type": "delay",
        "z": "da74aa24b406fbc1",
        "name": "Maximal alle 15min",
        "pauseType": "timed",
        "timeout": "5",
        "timeoutUnits": "seconds",
        "rate": "1",
        "nbRateUnits": "15",
        "rateUnits": "minute",
        "randomFirst": "1",
        "randomLast": "5",
        "randomUnits": "seconds",
        "drop": true,
        "allowrate": false,
        "outputs": 1,
        "x": 930,
        "y": 640,
        "wires": [
            [
                "52a5eba435b1e589"
            ]
        ]
    },
    {
        "id": "f161e82a45270adc",
        "type": "change",
        "z": "da74aa24b406fbc1",
        "name": "Zeit",
        "rules": [
            {
                "t": "set",
                "p": "payload",
                "pt": "msg",
                "to": "",
                "tot": "date"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 970,
        "y": 900,
        "wires": [
            [
                "52a5eba435b1e589",
                "d46609672595eacd",
                "63e6e930bd4811cf",
                "3ad98406e84e7654"
            ]
        ]
    },
    {
        "id": "ae2e308d25b37df2",
        "type": "inject",
        "z": "da74aa24b406fbc1",
        "name": "Alle 5 Minuten",
        "props": [
            {
                "p": "payload"
            },
            {
                "p": "topic",
                "vt": "str"
            }
        ],
        "repeat": "300",
        "crontab": "",
        "once": true,
        "onceDelay": 0.1,
        "topic": "timestamp",
        "payload": "",
        "payloadType": "date",
        "x": 600,
        "y": 900,
        "wires": [
            [
                "f161e82a45270adc"
            ]
        ]
    },
    {
        "id": "3cadef2b36f2b9ab",
        "type": "delay",
        "z": "da74aa24b406fbc1",
        "name": "Maximal alle 15min",
        "pauseType": "timed",
        "timeout": "5",
        "timeoutUnits": "seconds",
        "rate": "1",
        "nbRateUnits": "15",
        "rateUnits": "minute",
        "randomFirst": "1",
        "randomLast": "5",
        "randomUnits": "seconds",
        "drop": true,
        "allowrate": false,
        "outputs": 1,
        "x": 930,
        "y": 580,
        "wires": [
            [
                "52a5eba435b1e589"
            ]
        ]
    },
    {
        "id": "b41a2d6af5cb2e52",
        "type": "delay",
        "z": "da74aa24b406fbc1",
        "name": "Maximal alle 15min",
        "pauseType": "timed",
        "timeout": "5",
        "timeoutUnits": "seconds",
        "rate": "1",
        "nbRateUnits": "15",
        "rateUnits": "minute",
        "randomFirst": "1",
        "randomLast": "5",
        "randomUnits": "seconds",
        "drop": true,
        "allowrate": false,
        "outputs": 1,
        "x": 930,
        "y": 540,
        "wires": [
            [
                "52a5eba435b1e589"
            ]
        ]
    },
    {
        "id": "19282e7f290e0f87",
        "type": "delay",
        "z": "da74aa24b406fbc1",
        "name": "Maximal alle 15min",
        "pauseType": "timed",
        "timeout": "5",
        "timeoutUnits": "seconds",
        "rate": "1",
        "nbRateUnits": "15",
        "rateUnits": "minute",
        "randomFirst": "1",
        "randomLast": "5",
        "randomUnits": "seconds",
        "drop": true,
        "allowrate": false,
        "outputs": 1,
        "x": 930,
        "y": 500,
        "wires": [
            [
                "52a5eba435b1e589"
            ]
        ]
    },
    {
        "id": "db6aefa0851d099a",
        "type": "mqtt-broker",
        "name": "MQTT collector",
        "broker": "localhost",
        "port": "1883",
        "clientid": "",
        "autoConnect": true,
        "usetls": false,
        "protocolVersion": "4",
        "keepalive": "60",
        "cleansession": true,
        "birthTopic": "",
        "birthQos": "2",
        "birthRetain": "true",
        "birthPayload": "",
        "birthMsg": {},
        "closeTopic": "",
        "closeQos": "0",
        "closePayload": "",
        "closeMsg": {},
        "willTopic": "",
        "willQos": "0",
        "willPayload": "",
        "willMsg": {},
        "userProps": "",
        "sessionExpiry": ""
    }
]

There are still some questions open and I may ask deeper as soon as I have time to check and dig for solutions here - but my main question is answered and it works :smiley: - thank you again.
best regards
A. B.

1 Like
11

This topic was automatically closed 60 days after the last reply. New replies are no longer allowed.