Scraping http hosts

Hi there!.

I have this flow working on and i'm stuck on a node that is getting me sleepless.

[{"id":"1c91208.fec35df","type":"html","z":"13e3e683.083059","name":"","property":"payload","outproperty":"payload","tag":"tr","ret":"html","as":"multi","x":490,"y":680,"wires":[["24ff0fa2.0bd78"]]},{"id":"24ff0fa2.0bd78","type":"html","z":"13e3e683.083059","name":"","property":"payload","outproperty":"payload","tag":"td","ret":"html","as":"single","x":619,"y":680,"wires":[["dd64a4c0.568c58"]]},{"id":"99820d79.ce827","type":"http request","z":"13e3e683.083059","name":"","method":"GET","ret":"txt","paytoqs":"ignore","url":"http://ota.tasmota.com/tasmota/release/","tls":"","persist":false,"proxy":"","authType":"","x":350,"y":680,"wires":[["1c91208.fec35df"]]},{"id":"dd64a4c0.568c58","type":"switch","z":"13e3e683.083059","name":"","property":"payload[0]","propertyType":"msg","rules":[{"t":"cont","v":"tasmota.bin","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":750,"y":680,"wires":[["f16ce907.a37a88"]]},{"id":"f16ce907.a37a88","type":"change","z":"13e3e683.083059","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"availableVersion","tot":"str"},{"t":"set","p":"payload","pt":"msg","to":"payload[6]","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":924,"y":680,"wires":[["a7dbed0c.cc957"]]},{"id":"a7dbed0c.cc957","type":"join","z":"13e3e683.083059","name":"","mode":"custom","build":"object","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":true,"timeout":"","count":"2","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":1090,"y":720,"wires":[["d3fe8df0.e1e6e"]]},{"id":"d3fe8df0.e1e6e","type":"function","z":"13e3e683.083059","name":"compare tasmota web version with tasmota version","func":"var webVersion = msg.payload.availableVersion; //get web version\nvar deviceData = msg.payload.deviceData; //get array of devices\nvar current = [];//where results will go\nvar old = [];\n\n//loop through each device\nfor (let i = 0; i < deviceData.length; i++) {\n    let device = deviceData[i]; //get this device\n    var v = device.version; //get device version\n    device.version = v.split(\"(\")[0]; //split on \"(\" then take first element [0]\n    device.webVersion = webVersion; //optional\n    device.match = (device.version == device.webVersion); //compare device version to webVersion\n    if(device.match) {\n        current.push(device); //add result\n    } else {\n        old.push(device); //add result\n    }\n}\n\nmsg.payload = {\n    current: current,\n    old: old,\n}\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":1370,"y":680,"wires":[["88f9688.fd3c198","14555b50.363735"]]},{"id":"88f9688.fd3c198","type":"debug","z":"13e3e683.083059","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1670,"y":640,"wires":[]},{"id":"f8eb8e72.964b3","type":"function","z":"13e3e683.083059","name":"hosts","func":"const tasmotas = [\"192.168.1.40\",\"192.168.1.180\"]\nconst status= \"/cm?cmnd=Status%200\"\n\nfor(let x=0; x<tasmotas.length; x++){\n    node.send({ \n        url:\"http://\"+tasmotas[x]+status,\n        ip: tasmotas[x]\n    })\n}\n\nmsg.complete = true;\n//msg.payload = null;\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":330,"y":780,"wires":[["ec9d2b2e.071af8"]]},{"id":"5c4cede2.0021b4","type":"comment","z":"13e3e683.083059","name":"Check Tasmota Web Version","info":"","x":160,"y":600,"wires":[]},{"id":"bb624e81.2c073","type":"function","z":"13e3e683.083059","name":"fake device requests","func":"var fake_httprequests = {\n    \"192.168.1.40\": {\"Status\":{\"Module\":1,\"DeviceName\":\"luz_prueba\",\"FriendlyName\":[\"luz_prueba\"],\"Topic\":\"luz_prueba\",\"ButtonTopic\":\"0\",\"Power\":0,\"PowerOnState\":3,\"LedState\":1,\"LedMask\":\"FFFF\",\"SaveData\":1,\"SaveState\":1,\"SwitchTopic\":\"0\",\"SwitchMode\":[0,0,0,0,0,0,0,0],\"ButtonRetain\":0,\"SwitchRetain\":0,\"SensorRetain\":0,\"PowerRetain\":0},\"StatusPRM\":{\"Baudrate\":115200,\"SerialConfig\":\"8N1\",\"GroupTopic\":\"tasmotas\",\"OtaUrl\":\"http://ota.tasmota.com/tasmota/release/tasmota.bin\",\"RestartReason\":\"Power On\",\"Uptime\":\"14T20:22:27\",\"StartupUTC\":\"2021-01-05T21:28:02\",\"Sleep\":50,\"CfgHolder\":4617,\"BootCount\":304,\"BCResetTime\":\"2020-06-12T04:30:48\",\"SaveCount\":3440,\"SaveAddress\":\"F9000\"},\"StatusFWR\":{\"Version\":\"9.2.0(tasmota)\",\"BuildDateTime\":\"2020-12-21T15:03:40\",\"Boot\":7,\"Core\":\"2_7_4_9\",\"SDK\":\"2.2.2-dev(38a443e)\",\"CpuFrequency\":80,\"Hardware\":\"ESP8285\",\"CR\":\"404/699\"},\"StatusLOG\":{\"SerialLog\":2,\"WebLog\":2,\"MqttLog\":0,\"SysLog\":0,\"LogHost\":\"\",\"LogPort\":514,\"SSId\":[\"BellVille2.4ghz\",\"\"],\"TelePeriod\":300,\"Resolution\":\"558180C0\",\"SetOption\":[\"002A8009\",\"2805C8000100060000005A00000000000000\",\"00000228\",\"00006000\",\"00000000\"]},\"StatusMEM\":{\"ProgramSize\":586,\"Free\":416,\"Heap\":23,\"ProgramFlashSize\":1024,\"FlashSize\":1024,\"FlashChipId\":\"144051\",\"FlashFrequency\":40,\"FlashMode\":3,\"Features\":[\"00000809\",\"8FDAC787\",\"04368001\",\"000000CF\",\"010013C0\",\"C000F981\",\"00004004\",\"00001000\"],\"Drivers\":\"1,2,3,4,5,6,7,8,9,10,12,16,18,19,20,21,22,24,26,27,29,30,35,37,45\",\"Sensors\":\"1,2,3,4,5,6\"},\"StatusNET\":{\"Hostname\":\"luz_prueba-6101\",\"IPAddress\":\"192.168.1.40\",\"Gateway\":\"192.168.1.1\",\"Subnetmask\":\"255.255.255.0\",\"DNSServer\":\"192.168.1.1\",\"Mac\":\"60:01:94:CA:37:D5\",\"Webserver\":2,\"WifiConfig\":4,\"WifiPower\":17},\"StatusMQT\":{\"MqttHost\":\"192.168.1.8\",\"MqttPort\":1883,\"MqttClientMask\":\"DVES_%06X\",\"MqttClient\":\"DVES_CA37D5\",\"MqttUser\":\"mosquitto\",\"MqttCount\":4,\"MAX_PACKET_SIZE\":1200,\"KEEPALIVE\":30},\"StatusTIM\":{\"UTC\":\"2021-01-20T17:50:29\",\"Local\":\"2021-01-20T18:50:29\",\"StartDST\":\"2021-03-28T02:00:00\",\"EndDST\":\"2021-10-31T03:00:00\",\"Timezone\":\"+01:00\",\"Sunrise\":\"08:34\",\"Sunset\":\"17:28\"},\"StatusSNS\":{\"Time\":\"2021-01-20T18:50:29\"},\"StatusSTS\":{\"Time\":\"2021-01-20T18:50:29\",\"Uptime\":\"14T20:22:27\",\"UptimeSec\":1282947,\"Heap\":23,\"SleepMode\":\"Dynamic\",\"Sleep\":50,\"LoadAvg\":19,\"MqttCount\":4,\"POWER\":\"OFF\",\"Wifi\":{\"AP\":1,\"SSId\":\"BellVille2.4ghz\",\"BSSId\":\"C8:D7:19:52:9D:27\",\"Channel\":10,\"RSSI\":100,\"Signal\":-40,\"LinkCount\":4,\"Downtime\":\"0T00:09:52\"}}},\n    \"192.168.1.180\": {\"Status\":{\"Module\":1,\"DeviceName\":\"luz_escritorio_garaje\",\"FriendlyName\":[\"luz_escritorio_garaje\"],\"Topic\":\"luz_escritorio_garaje\",\"ButtonTopic\":\"0\",\"Power\":1,\"PowerOnState\":3,\"LedState\":1,\"LedMask\":\"FFFF\",\"SaveData\":1,\"SaveState\":1,\"SwitchTopic\":\"0\",\"SwitchMode\":[0,0,0,0,0,0,0,0],\"ButtonRetain\":0,\"SwitchRetain\":0,\"SensorRetain\":0,\"PowerRetain\":0},\"StatusPRM\":{\"Baudrate\":115200,\"SerialConfig\":\"8N1\",\"GroupTopic\":\"sonoffs\",\"OtaUrl\":\"http://thehackbox.org/tasmota/release/tasmota.bin\",\"RestartReason\":\"Software/System restart\",\"Uptime\":\"0T23:52:49\",\"StartupUTC\":\"2021-01-19T17:57:41\",\"Sleep\":50,\"CfgHolder\":4617,\"BootCount\":488,\"BCResetTime\":\"2020-04-15T00:57:39\",\"SaveCount\":6301,\"SaveAddress\":\"F7000\"},\"StatusFWR\":{\"Version\":\"8.5.0(tasmota)\",\"BuildDateTime\":\"2020-09-09T11:41:02\",\"Boot\":7,\"Core\":\"2_7_4_1\",\"SDK\":\"2.2.2-dev(38a443e)\",\"CpuFrequency\":80,\"Hardware\":\"ESP8285\",\"CR\":\"430/699\"},\"StatusLOG\":{\"SerialLog\":2,\"WebLog\":2,\"MqttLog\":0,\"SysLog\":0,\"LogHost\":\"\",\"LogPort\":514,\"SSId\":[\"BellVille2.4ghz\",\"\"],\"TelePeriod\":300,\"Resolution\":\"558180C0\",\"SetOption\":[\"002A8109\",\"2805C8000100060000005AFF000000000000\",\"00000200\",\"00006000\",\"00000000\"]},\"StatusMEM\":{\"ProgramSize\":595,\"Free\":408,\"Heap\":23,\"ProgramFlashSize\":1024,\"FlashSize\":1024,\"FlashChipId\":\"144051\",\"FlashFrequency\":40,\"FlashMode\":3,\"Features\":[\"00000809\",\"8FDAE797\",\"04368001\",\"000000CD\",\"010013C0\",\"C000F981\",\"00004004\",\"00000000\"],\"Drivers\":\"1,2,3,4,5,6,7,8,9,10,12,16,18,19,20,21,22,24,26,27,29,30,35,37\",\"Sensors\":\"1,2,3,4,5,6\"},\"StatusNET\":{\"Hostname\":\"luz_escritorio_garaje\",\"IPAddress\":\"192.168.1.180\",\"Gateway\":\"192.168.1.1\",\"Subnetmask\":\"255.255.255.0\",\"DNSServer\":\"192.168.1.1\",\"Mac\":\"DC:4F:22:F9:7D:69\",\"Webserver\":2,\"WifiConfig\":2,\"WifiPower\":17},\"StatusMQT\":{\"MqttHost\":\"192.168.1.8\",\"MqttPort\":1883,\"MqttClientMask\":\"DVES_%06X\",\"MqttClient\":\"DVES_F97D69\",\"MqttUser\":\"mosquitto\",\"MqttCount\":1,\"MAX_PACKET_SIZE\":1200,\"KEEPALIVE\":30},\"StatusTIM\":{\"UTC\":\"2021-01-20T17:50:30\",\"Local\":\"2021-01-20T18:50:30\",\"StartDST\":\"2021-03-28T02:00:00\",\"EndDST\":\"2021-10-31T03:00:00\",\"Timezone\":\"+01:00\",\"Sunrise\":\"08:34\",\"Sunset\":\"17:28\"},\"StatusSNS\":{\"Time\":\"2021-01-20T18:50:30\"},\"StatusSTS\":{\"Time\":\"2021-01-20T18:50:30\",\"Uptime\":\"0T23:52:49\",\"UptimeSec\":85969,\"Heap\":23,\"SleepMode\":\"Dynamic\",\"Sleep\":50,\"LoadAvg\":19,\"MqttCount\":1,\"POWER\":\"ON\",\"Wifi\":{\"AP\":1,\"SSId\":\"BellVille2.4ghz\",\"BSSId\":\"C8:D7:19:52:9D:27\",\"Channel\":10,\"RSSI\":94,\"Signal\":-53,\"LinkCount\":1,\"Downtime\":\"0T00:00:06\"}}},\n}\n\nmsg.payload = fake_httprequests[msg.ip]\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":500,"y":820,"wires":[[]]},{"id":"ec9d2b2e.071af8","type":"join","z":"13e3e683.083059","name":"","mode":"custom","build":"array","property":"payload","propertyType":"msg","key":"","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"","count":"","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":670,"y":780,"wires":[["17b0f4da.d1640b"]]},{"id":"17b0f4da.d1640b","type":"change","z":"13e3e683.083059","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"payload.{ \"name\":**.Hostname, \"ipAddress\":**.IPAddress, \"version\":**.Version }","tot":"jsonata"},{"t":"set","p":"topic","pt":"msg","to":"deviceData","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":920,"y":760,"wires":[["25481e0f.69bdf2","a7dbed0c.cc957"]]},{"id":"14555b50.363735","type":"function","z":"13e3e683.083059","name":"Send not updated devices to TG","func":"var oldDevices = msg.payload.old;\n\nif(oldDevices && oldDevices.length) {\n  msg.payload.chatId = 587232739;\n  msg.payload.type = 'message';\n  msg.payload.content = `Se encontraron ${oldDevices.length} dispositivo con la versión de Tasmota desactualizada:\\n` + msg.payload.old.map(e => `NOMBRE: ${e.name}, IP: ${e.ipAddress}, VER: ${e.version}`).join(\"\\n\")\n  return msg;\n}\n","outputs":1,"noerr":0,"initialize":"","finalize":"","x":1330,"y":740,"wires":[["236c15d.5dd1cea"]]},{"id":"8edcf171.30277","type":"comment","z":"13e3e683.083059","name":"fail","info":"No se está trayendo correctamente la version actual de los tasmota y eso es porque hay un nodo de faker que ya tiene los datos preestablecidos.\n","x":490,"y":860,"wires":[]},{"id":"25481e0f.69bdf2","type":"debug","z":"13e3e683.083059","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1130,"y":840,"wires":[]},{"id":"3113a0c9.91583","type":"inject","z":"13e3e683.083059","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":190,"y":780,"wires":[["f8eb8e72.964b3"]]}]

The issue is when i want to retrieve some the hosts information, i can fake it but i don't want because is not the real information.

How can i get the real Http information of every host so i can compare them later?.

Thanks a lot!

So this is a continueation of this thread from January right?

As far as I remember, all you had to do was drop the http request node in (that you had on your first/original flow) & it should of worked! (assuming the demo data you provided in Post#19 was correct)

So if you put the http request node from your original flow in place of the "fake device requests" function what happens?
PS: attach a debug to the ↑ http request node

Show us.

Cool @Steve-Mcl i could figured it out on one shot!. I replace the node you said and it worked like a charm!.

Thanks a lot again!

Reagards!

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.