Building an ioT app with Apache Kafka + MQTT

General
1

Hello there,

I would like some insights about the solution I am now implementing. My team is building an ioT app that monitors a certain industrial process. Our goal is to represent and show ioT information about sensors, actuators, motor velocities, etc.
We're using Node Red's node Apache Kafka (node-red-contrib-kafka-manager) to get information from the ioT platform. Is it a good practice to get that information using kafka and then send it to the front end web server using MQTT messages via aedes broker? Or eventually the application will get too heavy and slow? I searched and Kafka works well with thousands of devices (not the case), so is it a good practice?

Thanks

2 

[{"id":"ef92d72c.d2a878","type":"tab","label":"Testes kafka + mqtt messages","disabled":false,"info":""},{"id":"6ba9f419.fdbcdc","type":"Kafka Producer","z":"ef92d72c.d2a878","name":"","broker":"31298c08.cf5b74","topic":"jpm","topicSlash2dot":false,"requireAcks":1,"ackTimeoutMs":100,"partitionerType":0,"key":"","partition":0,"attributes":0,"connectionType":"Producer","convertFromJson":false,"x":700,"y":160,"wires":[]},{"id":"94335940.0b7e08","type":"Kafka Consumer","z":"ef92d72c.d2a878","name":"","broker":"31298c08.cf5b74","regex":false,"topics":[{"topic":"jpm","offset":0,"partition":0}],"groupId":"kafka-node-group","autoCommit":"true","autoCommitIntervalMs":5000,"fetchMaxWaitMs":100,"fetchMinBytes":1,"fetchMaxBytes":1048576,"fromOffset":0,"encoding":"utf8","keyEncoding":"utf8","connectionType":"Consumer","convertToJson":false,"x":370,"y":280,"wires":[["f1a2d195.819be"]]},{"id":"a9d290bb.bfb53","type":"inject","z":"ef92d72c.d2a878","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":360,"y":160,"wires":[["6ba9f419.fdbcdc"]]},{"id":"f1a2d195.819be","type":"debug","z":"ef92d72c.d2a878","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":580,"y":300,"wires":[]},{"id":"dd6e14e1.2bdb48","type":"inject","z":"ef92d72c.d2a878","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"5","payloadType":"num","x":1670,"y":200,"wires":[["ccd9a575.4f63f8"]]},{"id":"ccd9a575.4f63f8","type":"Kafka Producer","z":"ef92d72c.d2a878","name":"","broker":"44b7aa4c.212894","topic":"youtube","topicSlash2dot":false,"requireAcks":1,"ackTimeoutMs":100,"partitionerType":0,"key":"","partition":0,"attributes":0,"connectionType":"Producer","convertFromJson":false,"x":1960,"y":200,"wires":[]},{"id":"835e0e6c.621fe","type":"debug","z":"ef92d72c.d2a878","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1970,"y":280,"wires":[]},{"id":"f49f06ac.f53628","type":"kafka-consumer","z":"ef92d72c.d2a878","name":"","broker":"f82ed179.0a16c","outOfRangeOffset":"earliest","fromOffset":"latest","topic":"youtube","groupid":"","x":1660,"y":280,"wires":[["835e0e6c.621fe","88188b2c.3d95c8"]]},{"id":"88188b2c.3d95c8","type":"mqtt out","z":"ef92d72c.d2a878","name":"","topic":"topic/kafka","qos":"0","retain":"","respTopic":"","contentType":"","userProps":"","correl":"","expiry":"","broker":"f8577ad8.94ed88","x":1970,"y":320,"wires":[]},{"id":"31298c08.cf5b74","type":"Kafka Broker","name":"local","hosts":[{"host":"192.168.40.231","port":9092}],"hostsEnvVar":"","connectTimeout":"10000","requestTimeout":"30000","autoConnect":"true","idleConnection":"5","reconnectOnIdle":"true","maxAsyncRequests":"10","checkInterval":"10","selfSign":true,"usetls":false,"useCredentials":false},{"id":"44b7aa4c.212894","type":"Kafka Broker","name":"Kafta","hosts":[{"host":"192.168.40.231","port":9092}],"hostsEnvVar":"","connectTimeout":"1000","requestTimeout":"30000","autoConnect":"true","idleConnection":"5","reconnectOnIdle":"true","maxAsyncRequests":"10","checkInterval":"10","selfSign":true,"usetls":false,"useCredentials":false},{"id":"f82ed179.0a16c","type":"kafka-broker","name":"broker","hosts":"192.168.40.231:9092","selfsign":false,"usetls":false,"cacert":"","clientcert":"","privatekey":"","passphrase":""},{"id":"f8577ad8.94ed88","type":"mqtt-broker","name":"broker_new","broker":"ws://127.0.0.1:9000","port":"1883","clientid":"","usetls":false,"protocolVersion":"4","keepalive":"60","cleansession":true,"birthTopic":"","birthQos":"0","birthPayload":"","birthMsg":{},"closeTopic":"","closeQos":"0","closePayload":"","closeMsg":{},"willTopic":"","willQos":"0","willPayload":"","willMsg":{},"sessionExpiry":""}]
3

Hello. It is good practice to properly format flows and code in the forum... using the </> icon

See #3 - Get involved : Node-RED

4

Thanks, I am new here! I’ve already edited.

5

Nope :wink: It should look more like this...

[{"id":"ef92d72c.d2a878","type":"tab","label":"Testes kafka + mqtt messages","disabled":false,"info":""},{"id":"6ba9f419.fdbcdc","type":"Kafka Producer","z":"ef92d72c.d2a878","name":"","broker":"31298c08.cf5b74","topic":"jpm","topicSlash2dot":false,"requireAcks":1,"ackTimeoutMs":100,"partitionerType":0,"key":"","partition":0,"attributes":0,"connectionType":"Producer","convertFromJson":false,"x":700,"y":160,"wires":[]},{"id":"94335940.0b7e08","type":"Kafka Consumer","z":"ef92d72c.d2a878","name":"","broker":"31298c08.cf5b74","regex":false,"topics":[{"topic":"jpm","offset":0,"partition":0}],"groupId":"kafka-node-group","autoCommit":"true","autoCommitIntervalMs":5000,"fetchMaxWaitMs":100,"fetchMinBytes":1,"fetchMaxBytes":1048576,"fromOffset":0,"encoding":"utf8","keyEncoding":"utf8","connectionType":"Consumer","convertToJson":false,"x":370,"y":280,"wires":[["f1a2d195.819be"]]},{"id":"a9d290bb.bfb53","type":"inject","z":"ef92d72c.d2a878","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":360,"y":160,"wires":[["6ba9f419.fdbcdc"]]},{"id":"f1a2d195.819be","type":"debug","z":"ef92d72c.d2a878","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":580,"y":300,"wires":[]},{"id":"dd6e14e1.2bdb48","type":"inject","z":"ef92d72c.d2a878","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"5","payloadType":"num","x":1670,"y":200,"wires":[["ccd9a575.4f63f8"]]},{"id":"ccd9a575.4f63f8","type":"Kafka Producer","z":"ef92d72c.d2a878","name":"","broker":"44b7aa4c.212894","topic":"youtube","topicSlash2dot":false,"requireAcks":1,"ackTimeoutMs":100,"partitionerType":0,"key":"","partition":0,"attributes":0,"connectionType":"Producer","convertFromJson":false,"x":1960,"y":200,"wires":[]},{"id":"835e0e6c.621fe","type":"debug","z":"ef92d72c.d2a878","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1970,"y":280,"wires":[]},{"id":"f49f06ac.f53628","type":"kafka-consumer","z":"ef92d72c.d2a878","name":"","broker":"f82ed179.0a16c","outOfRangeOffset":"earliest","fromOffset":"latest","topic":"youtube","groupid":"","x":1660,"y":280,"wires":[["835e0e6c.621fe","88188b2c.3d95c8"]]},{"id":"88188b2c.3d95c8","type":"mqtt out","z":"ef92d72c.d2a878","name":"","topic":"topic/kafka","qos":"0","retain":"","respTopic":"","contentType":"","userProps":"","correl":"","expiry":"","broker":"f8577ad8.94ed88","x":1970,"y":320,"wires":[]},{"id":"31298c08.cf5b74","type":"Kafka Broker","name":"local","hosts":[{"host":"192.168.40.231","port":9092}],"hostsEnvVar":"","connectTimeout":"10000","requestTimeout":"30000","autoConnect":"true","idleConnection":"5","reconnectOnIdle":"true","maxAsyncRequests":"10","checkInterval":"10","selfSign":true,"usetls":false,"useCredentials":false},{"id":"44b7aa4c.212894","type":"Kafka Broker","name":"Kafta","hosts":[{"host":"192.168.40.231","port":9092}],"hostsEnvVar":"","connectTimeout":"1000","requestTimeout":"30000","autoConnect":"true","idleConnection":"5","reconnectOnIdle":"true","maxAsyncRequests":"10","checkInterval":"10","selfSign":true,"usetls":false,"useCredentials":false},{"id":"f82ed179.0a16c","type":"kafka-broker","name":"broker","hosts":"192.168.40.231:9092","selfsign":false,"usetls":false,"cacert":"","clientcert":"","privatekey":"","passphrase":""},{"id":"f8577ad8.94ed88","type":"mqtt-broker","name":"broker_new","broker":"ws://127.0.0.1:9000","port":"1883","clientid":"","usetls":false,"protocolVersion":"4","keepalive":"60","cleansession":true,"birthTopic":"","birthQos":"0","birthPayload":"","birthMsg":{},"closeTopic":"","closeQos":"0","closePayload":"","closeMsg":{},"willTopic":"","willQos":"0","willPayload":"","willMsg":{},"sessionExpiry":""}]