Generic sensor processing code

[Colin]

@bkenobi

Is there any way to have a lookup so that I could have a list of sensor ID's and a meaningful name for them to be associated with in the database?

Yes, lots of different ways of doing it. The simplest is just to do it in a function node, so going from the topic, for example, following on from what I showed you earlier, you could do

const table = [
  {id: 245, tag: "North Attic"},
  {id: 251, tag: "South Attic"},
]
const topicArray = msg.topic.split("/")
const id = topicArray[topicArray.length - 1]
// now use table.filter() to find lookup the appropriate one

I haven't got time to show you how to do that, google for javascript Array.filter and see if you can work it out.

[bkenobi]

Appreciate the guidance!

[craigcurtin]

You could also just use the switch node and have it look at the ID property of the object - one large switch node with each of the IDs listed and an otherwise catch all at the end that then sends it out to some form of file storage for ones that are not caught.

Once you switch on ID then you can do your influx storage as required per device

Craig

[bkenobi]

I have a splitter and change node in series that now split the json into different ports such that I can route them as needed. I've been looking off and on since yesterday on how to utilize the array filter function but haven't found anything that looks correct. Here's a shortened version of what I currently have.

[{"id":"9d1cdbd1.d25ed","type":"function","z":"a3c4e269.ff5da","name":"id to location","func":"const table = [\n  {id: 1066, tag: \"B\"},\n  {id: 245, tag: \"A\"},\n  {id: 1878, tag: \"C\"},\n  {id: 9170, tag: \"D\"},\n  {id: 11106, tag: \"E\"},\n  {id: 1294, tag: \"F\"},\n  {id: 10160, tag: \"G\"},\n]\nconst topicArray = msg.topic.split(\"/\")\nconst id = topicArray[topicArray.length - 1]\n// now use table.filter() to find lookup the appropriate one\nmsg.payload = id\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":690,"y":400,"wires":[[]]},{"id":"b890bc86.1bbde8","type":"debug","z":"a3c4e269.ff5da","name":"humidity","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":580,"y":520,"wires":[]},{"id":"9f28c1f6.cb06a","type":"debug","z":"a3c4e269.ff5da","name":"battery","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":570,"y":560,"wires":[]},{"id":"fefaec1e.dadd8","type":"influxdb out","z":"a3c4e269.ff5da","d":true,"influxdb":"5b91dbad.94ba7c","name":"TempF","measurement":"","precision":"","retentionPolicy":"","database":"sensors","precisionV18FluxV20":"ms","retentionPolicyV18Flux":"","org":"organisation","bucket":"bucket","x":770,"y":480,"wires":[]},{"id":"9100e02d.7b5668","type":"switch","z":"a3c4e269.ff5da","name":"by topic","property":"topic","propertyType":"msg","rules":[{"t":"eq","v":"id","vt":"str"},{"t":"eq","v":"model","vt":"str"},{"t":"eq","v":"temperature_C","vt":"str"},{"t":"eq","v":"humidity","vt":"str"},{"t":"eq","v":"battery_ok","vt":"str"}],"checkall":"true","repair":false,"outputs":5,"x":380,"y":460,"wires":[["b5e0b53a.b47108"],["9ce3b83c.884cb"],["7d954704.7195f"],["b890bc86.1bbde8"],["9f28c1f6.cb06a"]]},{"id":"a1a50547.4cf42","type":"split","z":"a3c4e269.ff5da","name":"splitter","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"topic","x":230,"y":460,"wires":[["9100e02d.7b5668"]]},{"id":"bc96ace8.cf89b","type":"mqtt in","z":"a3c4e269.ff5da","name":"MQTT","topic":"rtl_433/Acurite-Tower/#","qos":"2","datatype":"json","broker":"b2400571.717f88","x":90,"y":460,"wires":[["d812ce4c.2bc208","a1a50547.4cf42"]]},{"id":"7d954704.7195f","type":"unit-converter","z":"a3c4e269.ff5da","category":"temperature","inputUnit":"C","outputUnit":"F","inputField":"payload","outputField":"payload","inputFieldType":"msg","outputFieldType":"msg","roundOutputField":false,"outputFieldDecimals":2,"name":"TempC2F","x":580,"y":480,"wires":[["fefaec1e.dadd8","1ffafa31.1223ee"]]},{"id":"1ffafa31.1223ee","type":"debug","z":"a3c4e269.ff5da","name":"temperature","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":790,"y":440,"wires":[]},{"id":"d812ce4c.2bc208","type":"debug","z":"a3c4e269.ff5da","name":"MQTT debug","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":250,"y":380,"wires":[]},{"id":"9ce3b83c.884cb","type":"debug","z":"a3c4e269.ff5da","name":"model","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":570,"y":440,"wires":[]},{"id":"5dd4e5ca.3628b4","type":"deduplicate","z":"a3c4e269.ff5da","name":"","keyproperty":"","expirypolicy":"keep","registryclass":"","expiry":"1","x":530,"y":400,"wires":[["9d1cdbd1.d25ed"],[]]},{"id":"b5e0b53a.b47108","type":"debug","z":"a3c4e269.ff5da","name":"id","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":510,"y":360,"wires":[]},{"id":"5b91dbad.94ba7c","type":"influxdb","hostname":"127.0.0.1","port":"8086","protocol":"http","database":"sensors","name":"influxdb","usetls":false,"tls":"","influxdbVersion":"1.x","url":"http://localhost:8086","rejectUnauthorized":true},{"id":"b2400571.717f88","type":"mqtt-broker","name":"RPi3","broker":"192.168.0.200","port":"1883","clientid":"","usetls":false,"compatmode":false,"keepalive":"60","cleansession":true,"birthTopic":"","birthQos":"0","birthPayload":"","closeTopic":"","closeQos":"0","closePayload":"","willTopic":"","willQos":"0","willPayload":""}]

My main issues seem to be:

1. how to get the id converted to a location
2. how to use the id/location value and sensor type to generate an influxdb injection node

From what has been suggested here and further reviewing what other people are doing with databases it appears that the best practice is to use tags rather than a single key/value pair like I have currently. I gather that I should use a setup similar to:

tags
	sensor type
	location or id
fields
	temp
	humidity
	battery
	wind speed
	wind direction
	rain

If that's correct, I still don't see how to accomplish this with the standard Influxdb inject node.

[Colin]

@bkenobi

I still don't see how to accomplish this with the standard Influxdb inject node.

To quote from the help text for the node:
If msg.payload is an array containing two objects, the first object will be written as the set of named fields, the second is the set of named tags.

So msg.payload needs to be something like

[
  {temp: 75, humidity: 80, ...},
  {sensor: "some sensor", location: "some location"}
]

What is the sensor tag for?

[bkenobi]

The node I had been using to send data to influx was the default one I found in another guide. I wasn't sure how to use the batch inject node but now that I see what you wrote and what the guide shows it makes sense. I need to build a message with the temp, humidity, etc as keys and the location and possibly sensor as tags.

I'm still not sure how I want to organize things. I was thinking each sensor should have the values written associated with it. But I've also considered the data type as the measurement and then having a tag for the location and perhaps the sensor type.

temperature
   N attic
   S attic
humidity
   N attic
   S attic

versus

N attic
   temp
   humidity
S attic
   temp
   humidity

I don't know if this is overthinking it though since with a database you can simply create a SQL query and either way you should get what is needed.

[Colin]

@bkenobi

The node I had been using to send data to influx was the default one I found in another guide. I wasn't sure how to use the batch inject node but now that I see what you wrote and what the guide shows it makes sense.

No, not the batch node, the normal influxdb Out node.

@bkenobi

I was thinking each sensor should have the values written associated with it.

Is that not the location? If not then what do you mean by sensor?

[bkenobi]

When writing out the above in a payload, I don't see how I would even do it the first way. So this is the data structure.

msg.payload = [
    {
        measurement: "Acurite_Tower",
        fields: {
            temp: 55,
            humidity: 51
        },
        tags:{
            location:"NorthAttic"
        },
        timestamp: new Date()
    }

This all happens so fast that I don't see a whole lot of need to add extra code to preserve the MQTT delivered timestamp if that's hard. I haven't looked at that yet. I'm already taking bigger bites at a time than I should.

[bkenobi]

Or should measurement be temperature, humidity, etc and then the field would be the sensor location?

[Colin]

The payload can be exactly as I showed.

[
  {temp: 75, humidity: 80, ...},
  {sensor: "some sensor", location: "some location"}
]

They all go to the same measurement. At least all the sensors of that type. You need a different measurement for the weather sensor as the fields are different. Also I don't think you need the sensor tag at all. What does it tell you that location does not?

Temperature, humitidy, etc are fields, location is a tag.

[bkenobi]

Oh, ok. So you are suggesting using the standard influxdb node and having it write to a measurement (e.g., Acurite_Tower) with the content being what you posted and that would result in tags "sensor" and "location" with keys "temp" and "humidity". And since there's really no need for a tag for sensor type, that could be excluded. My weather station would be written in a similar way but go to a measurement "weather_station" with keys "temp", "humidity", "wind_speed", etc. and an optional tag for type and/or location.

To build that payload I'm assuming the best way is to use a function node with multiple inputs and then just build the payload.

[Colin]

@bkenobi

Oh, ok. So you are suggesting using the standard influxdb node and having it write to a measurement (e.g., Acurite_Tower) with the content being what you posted and that would result in tags "sensor" and "location" with keys "temp" and "humidity". And since there's really no need for a tag for sensor type, that could be excluded. My weather station would be written in a similar way but go to a measurement "weather_station" with keys "temp", "humidity", "wind_speed", etc. and an optional tag for type and/or location.

Yes, but they are fields, not keys.

@bkenobi

function node with multiple inputs

Why multiple inputs? You said all the values come from the device in one message.

[bkenobi]

My understanding of how to use NR in general is that you use a node to bring in the data and then use other nodes to process it. I could use a single function node to do everything but isn't that counter to the "proper" way to use NR? This is all a learning experience to which I'm grateful!

[bkenobi]

For example, my current flow for processing my weather station is:

Changing to a single json message should simplify. This isn't working yet but my initial conversion pending more work:

But if I have to combine back together in a function, I don't see why I wouldn't do the majority of this in that same function which is against the spirit of NR as I understand.

[Colin]

There is no need to split up the message coming from the device till you need to. Convert it to a javascript object containing all the values and give that to MQTT. Then you can pick that object up from MQTT, do a simple manipulation of that and feed it direct to the influx node. When you need to split it you can do that, but very often it is not necessary to split it. Most of the the node red dashboard nodes, for example, can be told which property to display, it is not necessary to split it out.

In the case of writing to influx it certainly makes no sense to split it and then go to effort of trying to join it all back together again (which is often not as simple as you might think).

[Edit] I assumed above that you wanted to use MQTT, but I am not sure what the second image you showed is for. If you are not using MQTT then that doesn't affect the principle.

[bkenobi]

I think I have the core of what I need to accomplish written in a simpler form but I am unclear how to get it into the data structure you have recommended.

I can use this test flow to convert the temperature to degF and delete the unneeded portions of the object.

[{"id":"85a995d5.f6c528","type":"unit-converter","z":"a3c4e269.ff5da","category":"temperature","inputUnit":"C","outputUnit":"F","inputField":"payload.temperature_C","outputField":"payload.tempF","inputFieldType":"msg","outputFieldType":"msg","roundOutputField":false,"outputFieldDecimals":2,"name":"TempC2F","x":240,"y":600,"wires":[["826546fe.ac77a8"]]},{"id":"826546fe.ac77a8","type":"change","z":"a3c4e269.ff5da","name":"","rules":[{"t":"delete","p":"payload.temperature_C","pt":"msg"},{"t":"delete","p":"payload.channel","pt":"msg"},{"t":"delete","p":"payload.mic","pt":"msg"},{"t":"delete","p":"payload.time","pt":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":420,"y":600,"wires":[["80015f63.9feea"]]},{"id":"ed9ca2e0.808f5","type":"inject","z":"a3c4e269.ff5da","name":"245","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"rtl_433/Acurite-Tower/245","payload":"{\"time\":\"2022-04-04 14:56:17\",\"model\":\"Acurite-Tower\",\"id\":245,\"channel\":\"A\",\"battery_ok\":0,\"temperature_C\":29.4,\"humidity\":23,\"mic\":\"CHECKSUM\"}","payloadType":"json","x":90,"y":600,"wires":[["85a995d5.f6c528","80015f63.9feea"]]},{"id":"80015f63.9feea","type":"debug","z":"a3c4e269.ff5da","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":370,"y":520,"wires":[]}]

The location lookup was not as simple for me as I don't have a JS background, but after a lot of searching and tinkering, the following does work.

[{"id":"6f8ab3a3.5dbba4","type":"debug","z":"a3c4e269.ff5da","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":450,"y":680,"wires":[]},{"id":"5ab8067e.d7004","type":"function","z":"a3c4e269.ff5da","name":"id to location","func":"const table = [\n  {id: 245, tag: \"A\"},\n  {id: 1066, tag: \"B\"},\n  {id: 1878, tag: \"C\"},\n  {id: 9170, tag: \"D\"},\n  {id: 11106, tag: \"E\"},\n  {id: 1294, tag: \"F\"},\n  {id: 10160, tag: \"G\"},\n]\nconst topicArray = msg.topic.split(\"/\")\nconst id = topicArray[topicArray.length - 1]\n\nmsg.payload = table.filter(table => table.id==id)[0].tag;\n//msg.payload = o.tag;\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":290,"y":740,"wires":[["6f8ab3a3.5dbba4"]]},{"id":"42f9dcf8.c15b94","type":"inject","z":"a3c4e269.ff5da","name":"245","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"rtl_433/Acurite-Tower/245","payload":"{\"time\":\"2022-04-04 14:56:17\",\"model\":\"Acurite-Tower\",\"id\":245,\"channel\":\"A\",\"battery_ok\":0,\"temperature_C\":29.4,\"humidity\":23,\"mic\":\"CHECKSUM\"}","payloadType":"json","x":90,"y":700,"wires":[["5ab8067e.d7004","6f8ab3a3.5dbba4"]]},{"id":"95a8f475.18f258","type":"inject","z":"a3c4e269.ff5da","name":"1878","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"rtl_433/Acurite-Tower/1878","payload":"{\"time\":\"2022-04-04 14:56:16\",\"model\":\"Acurite-Tower\",\"id\":1878,\"channel\":\"C\",\"battery_ok\":1,\"temperature_C\":19.7,\"humidity\":41,\"mic\":\"CHECKSUM\"}","payloadType":"json","x":90,"y":780,"wires":[["5ab8067e.d7004"]]},{"id":"db7f0a3a.abf488","type":"inject","z":"a3c4e269.ff5da","name":"1066","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"rtl_433/Acurite-Tower/1066","payload":"{\"time\":\"2022-04-04 14:56:15\",\"model\":\"Acurite-Tower\",\"id\":1066,\"channel\":\"B\",\"battery_ok\":1,\"temperature_C\":20.4,\"humidity\":41,\"mic\":\"CHECKSUM\"}","payloadType":"json","x":90,"y":740,"wires":[["5ab8067e.d7004"]]}]

I am not sure how to get my existing json object appended to the tag with location.

[bkenobi]

It doesn't give the right form, but I can obviously add the tag by just adding the tag as a new key to msg.payload. But again, that doesn't match the format so I'm sure when I submit to Influx won't work as desired.

const table = [
  {id: 245, tag: "A"},
  {id: 1066, tag: "B"},
  {id: 1878, tag: "C"},
  {id: 9170, tag: "D"},
  {id: 11106, tag: "E"},
  {id: 1294, tag: "F"},
  {id: 10160, tag: "G"},
]
const topicArray = msg.topic.split("/")
const id = topicArray[topicArray.length - 1]

msg.payload.tag = table.filter(table => table.id==id)[0].tag;

return msg;

[craigcurtin]

Use a change node and have it put the value in that you want to the object

The Change node can just add a field to an object if it is already there

I have not followed along since Colin stepped in - but are you saying you do not want a field in the object with Location or get rid of the Channel field ?

Craig

[bkenobi]

My goal is simply to take the data from the sensor, convert the temperature to F, and store temp/humidity to influx. Based on Colin's recommendation, I was trying to inject a location in such a way that influx could use it as a filterable item. Colin recommended that being done by adding a tag. The code he provided and I flushed out takes a list of sensor id's and converts that to a location (or pseudo channel in the current test code). I think I just need help figuring out how to make the payload into the format that influx needs to store correctly. I don't know how to make it look like Colin recommended though:

[
  {temp: 75, humidity: 80, ...},
  {sensor: "some sensor", location: "some location"}
]

[bkenobi]

I thought I could just build the payload like this, but it doesn't work.

var newmsg = { topic: "Acurite-Tower/"  & id};
newmsg = { payload: "[{tempF: " & msg.payload.tempF & ", humidity: " & msg.payload.humidity & ", battery_ok: " & msg.payload.battery_ok & "}, {location: " & msg.payload.tag & "}]"};

[{"id":"85a995d5.f6c528","type":"unit-converter","z":"a3c4e269.ff5da","category":"temperature","inputUnit":"C","outputUnit":"F","inputField":"payload.temperature_C","outputField":"payload.tempF","inputFieldType":"msg","outputFieldType":"msg","roundOutputField":false,"outputFieldDecimals":2,"name":"TempC2F","x":240,"y":600,"wires":[["826546fe.ac77a8"]]},{"id":"6f8ab3a3.5dbba4","type":"debug","z":"a3c4e269.ff5da","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":670,"y":600,"wires":[]},{"id":"826546fe.ac77a8","type":"change","z":"a3c4e269.ff5da","name":"","rules":[{"t":"delete","p":"payload.temperature_C","pt":"msg"},{"t":"delete","p":"payload.channel","pt":"msg"},{"t":"delete","p":"payload.mic","pt":"msg"},{"t":"delete","p":"payload.time","pt":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":420,"y":600,"wires":[["5ab8067e.d7004","6f8ab3a3.5dbba4"]]},{"id":"5ab8067e.d7004","type":"function","z":"a3c4e269.ff5da","name":"id to location","func":"const table = [\n  {id: 245, tag: \"A\"},\n  {id: 1066, tag: \"B\"},\n  {id: 1878, tag: \"C\"},\n  {id: 9170, tag: \"D\"},\n  {id: 11106, tag: \"E\"},\n  {id: 1294, tag: \"F\"},\n  {id: 10160, tag: \"G\"},\n]\nconst topicArray = msg.topic.split(\"/\")\nconst id = topicArray[topicArray.length - 1]\n\nmsg.payload.tag = table.filter(table => table.id==id)[0].tag;\nvar newmsg = { topic: \"Acurite-Tower/\"  & id};\nnewmsg = { payload: \"[{tempF: \" & msg.payload.tempF & \", humidity: \" & msg.payload.humidity & \", battery_ok: \" & msg.payload.battery_ok & \"}, {location: \" & msg.payload.tag & \"}]\"};\n\nreturn [msg, newmsg];\n//return msg;","outputs":2,"noerr":0,"initialize":"","finalize":"","x":550,"y":660,"wires":[["6f8ab3a3.5dbba4"],["6f8ab3a3.5dbba4"]]},{"id":"ed9ca2e0.808f5","type":"inject","z":"a3c4e269.ff5da","name":"245","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"rtl_433/Acurite-Tower/245","payload":"{\"time\":\"2022-04-04 14:56:17\",\"model\":\"Acurite-Tower\",\"id\":245,\"channel\":\"A\",\"battery_ok\":0,\"temperature_C\":29.4,\"humidity\":23,\"mic\":\"CHECKSUM\"}","payloadType":"json","x":90,"y":560,"wires":[["85a995d5.f6c528"]]},{"id":"95a8f475.18f258","type":"inject","z":"a3c4e269.ff5da","name":"1878","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"rtl_433/Acurite-Tower/1878","payload":"{\"time\":\"2022-04-04 14:56:16\",\"model\":\"Acurite-Tower\",\"id\":1878,\"channel\":\"C\",\"battery_ok\":1,\"temperature_C\":19.7,\"humidity\":41,\"mic\":\"CHECKSUM\"}","payloadType":"json","x":90,"y":640,"wires":[["85a995d5.f6c528"]]},{"id":"db7f0a3a.abf488","type":"inject","z":"a3c4e269.ff5da","name":"1066","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"rtl_433/Acurite-Tower/1066","payload":"{\"time\":\"2022-04-04 14:56:15\",\"model\":\"Acurite-Tower\",\"id\":1066,\"channel\":\"B\",\"battery_ok\":1,\"temperature_C\":20.4,\"humidity\":41,\"mic\":\"CHECKSUM\"}","payloadType":"json","x":90,"y":600,"wires":[["85a995d5.f6c528"]]}]

The payload is "0" instead of the object I expected.