Newly refreshed dashboard table shows stale data after all dashboards closed for some time

[GeorgeCastanza]

I've got a flow that records the last time any zigbee2mqtt device meshes with the zigbee mesh, to check if any devices have dropped off.

The problem is that the dash table only updates if the dashboard is open somewhere. First thing in the morning (for example), when I open the dashboard, the table in question has last updated about the time I shut down my computer browser the previous evening. I'm unsure as to the reason why.

[{"id":"87af4aff.b17178","type":"change","z":"8c249630.076968","name":"Make table update","rules":[{"t":"set","p":"payload","pt":"msg","to":"{\t   \"command\":\"updateOrAddData\",\t   \"arguments\":[\t       [\t           {\t               \"id\":msg.device.friendly_name,\t               \"value\":msg.payload,\t               \"name\":msg.device.friendly_name\t            }\t        ]\t    ],\t   \"returnPromise\":true\t}","tot":"jsonata"}],"action":"","property":"","from":"","to":"","reg":false,"x":1750,"y":1640,"wires":[["2c22ee36.5a7e52","e32ca362.ba5fa"]]},{"id":"e32ca362.ba5fa","type":"function","z":"8c249630.076968","name":"table recorder","func":"var status = {fill:\"red\",shape:\"ring\",text:\"an error occured\"};\nvar success = (msg.topic && msg.topic===\"success\") || false;\nvar tableData2 = flow.get(\"tableData2\");\nif (tableData2 === undefined) {\n    tableData2 = [];\n    flow.set(\"tableData2\",tableData2);\n}\n\n// find the index for a row in tableData2 for a given index (id)\nfunction checkIndex(id) {\n    let matchRow=-1;\n    tableData2.forEach(function (row,index){\n        if (row.id === id){\n            matchRow=index;\n            return matchRow;\n        }\n    });\n    return matchRow;\n}\n\n// flat merge one row \nfunction mergeRow(dest,source) {\n    Object.keys(source).forEach(function(key) {\n        dest[key]=source[key];\n    });\n}\n\n//merge or add one or many rows into tableData2 \nfunction mergeData(newData,toTop) {\n    newData.forEach(function (item,index) {\n     //   node.warn([\"findIndex\",item]);\n        let row=checkIndex(item.id);\n        if (row<0) { // row do not existst in tableData2\n            if (toTop) {\n                tableData2.push(item);\n                status.text+=\"newRow @ top\";\n            } else {\n                tableData2.unshift(item);\n                status.text+=\"newRow @ bottom\";\n            }\n            return;\n        } else { // row exists so update\n            mergeRow(tableData2[row],item);\n            status.text+=\"row updated\";\n            return;\n        }\n        if (status.text!==\"\") node.status(status);\n    });\n}\n\nswitch (typeof msg.payload){\n    case \"string\":\n     //   node.warn([\"[table recorder] \",(typeof msg.payload),msg.payload]);\n        switch (msg.payload){\n            case \"change\":\n                status={fill:\"green\",shape:\"dot\",text:\"table restored \"+tableData2.length+\" rows\"};\n                msg.payload=tableData2;\n                break;\n        }\n        break;\n    case \"object\":\n       // node.warn([\"[table recorder] \",(typeof msg.payload),msg.payload]);\n        if (Array.isArray(msg.payload)) { // replace all tableData2\n            status={fill:\"green\",shape:\"dot\",text:\"table replaced \"+msg.payload.length+\" rows\"};\n            tableData2=RED.util.cloneMessage(msg.payload); \n        } else {\n            switch (msg.payload.command) { // clearData does not return a promise!\n                case \"clearData\":\n                    status={fill:\"green\",shape:\"dot\",text:\"clearData: done\"};\n                    tableData2=[];\n                    flow.set(\"lastId\",0);\n                    break;                \n            }\n        }\n        break;\n    default: // likely a msg fom a ui-table command or callback\n        if (msg.hasOwnProperty(\"topic\")&&\n            msg.hasOwnProperty(\"ui_control\") && \n            msg.ui_control.hasOwnProperty(\"callback\") &&\n            msg.hasOwnProperty(\"return\")) { // message originates from a ui-table callback\n            if (success) {\n                switch(msg.return.command) {\n                    case \"addRow\":\n                        status.text=\"addRow: \";\n                        mergeData(msg.return.arguments[0],msg.return.arguments[1]);\n                        status.shape=\"dot\";\n                        break;\n                    case \"updateOrAddData\":\n                        status.text=\"updateOrAddData: \";\n                        mergeData(msg.return.arguments[0]);\n                        break;\n                    case \"deleteRow\":\n                        let row=checkIndex(msg.return.arguments[0]);\n                        tableData2.splice(row,1);\n                        status.shape=\"dot\";\n                        status.text=\"deleteRow: \"+row+\" deleted\";\n                        break;\n                    default:\n                        status={fill:\"yellow\",shape:\"dot\",text:msg.return.command + \" unknown!\"};\n                        break;         \n                }\n            } else {\n                status.text=msg.topic+\" \"+msg.error;\n            }\n        }\n        break;\n}\nif (success) status.fill=\"green\";\nflow.set(\"tableData2\",tableData2);\nnode.status(status);\nreturn msg;\n","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":2080,"y":1700,"wires":[["2c22ee36.5a7e52"]],"icon":"font-awesome/fa-database","info":"# simple ui-table handler\n## abstract\nUsing ui-table with commands offer the hole flexibilty of tabulator. The table can be manipulated down to cell level.\nAs the ui-table node only passes the commands to tabulator and receives promises back the node does not hold the table data. If the data should be available after refresh, tab change, new connections the flow is responsible to cache the data and all the manipulations.\nThis node takes care of most simple data manipulation commands and holds a copy of the data in `flow.context.tabledata`\n\n## details\n\n### row index (id)\n\nTo identify a [row a index](http://tabulator.info/docs/4.5/data#overview) column has to be defined. This colum defaults to `id` but can be changed by specifing a **field** by using `msg.ui_control`. In this example the row index is a simple counter adding up by one if a new line is added.\n\n### addRow command\n\n[details @ tabulator addRow docs](http://tabulator.info/docs/4.5/update#alter-add)\n\nYou can add a row by sending the `addRow` command. You can decide if the row adds on the top or at the bottom of table.\n\n### addOrUpdate command\n\n[details @ tabulator addOrUpdate docs](http://tabulator.info/docs/4.5/update#alter-update)\n\nTo update data the best way is to use the `addOrUpdate` command. If the row indetified by the index is not exeisting a new row will be added automatically\n\n### deleteRow command\n\n[details @ tabulator deleteRow docs](http://tabulator.info/docs/4.5/update#row)\n\nDelete one or more rows (passing an array always results in \"row not found error\"! I think there is an issue in tabulator)\n\n### clearData\n\n[details @ tabulator clearData docs](http://tabulator.info/docs/4.5/update#alter-empty)\n\nunfortunately this command (currently) do not send a promise back! So we have to pass it directly to the table handler"},{"id":"2c22ee36.5a7e52","type":"ui_table","z":"8c249630.076968","group":"aa839120.af379","name":"Table","order":2,"width":"7","height":"34","columns":[{"field":"name","title":"Name","width":"50%","align":"left","formatter":"plaintext","formatterParams":{"target":"_blank"}},{"field":"value","title":"Last Seen","width":"50%","align":"left","formatter":"plaintext","formatterParams":{"target":"_blank"}}],"outputs":1,"cts":true,"x":2090,"y":1620,"wires":[["e32ca362.ba5fa"]]},{"id":"627f575b.807ba8","type":"ui_ui_control","z":"8c249630.076968","name":"","events":"all","x":1860,"y":1700,"wires":[["e32ca362.ba5fa"]]},{"id":"7809fd85.767624","type":"moment","z":"8c249630.076968","name":"Format","topic":"","input":"payload","inputType":"msg","inTz":"Australia/Sydney","adjAmount":0,"adjType":"days","adjDir":"add","format":"LLL","locale":"en-US","output":"payload","outputType":"msg","outTz":"Australia/Sydney","x":1580,"y":1640,"wires":[["87af4aff.b17178"]]},{"id":"23d0d89d.ce8c78","type":"change","z":"8c249630.076968","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"device.friendly_name","tot":"msg"},{"t":"set","p":"payload","pt":"msg","to":"$now()","tot":"jsonata"}],"action":"","property":"","from":"","to":"","reg":false,"x":1260,"y":1640,"wires":[["be50a728cbf07ce7"]]},{"id":"27ccfd14.9342e2","type":"zigbee2mqtt-in","z":"8c249630.076968","name":"","server":"7b626d8c.6f0df4","friendly_name":"Temp - Kitchen","device_id":"0x00158d00016c6081","state":"0","outputAtStartup":true,"x":940,"y":1640,"wires":[["23d0d89d.ce8c78"]],"info":"Not working, as the profile is not setup in zibgee2mqtt"},{"id":"b32ac3e1.6b7dc","type":"zigbee2mqtt-in","z":"8c249630.076968","name":"","server":"7b626d8c.6f0df4","friendly_name":"Temp - Outside Down","device_id":"0x00158d0004876b5e","state":"0","outputAtStartup":true,"x":920,"y":1700,"wires":[["23d0d89d.ce8c78"]],"info":"Not working, as the profile is not setup in zibgee2mqtt"},{"id":"be50a728cbf07ce7","type":"switch","z":"8c249630.076968","name":"","property":"topic","propertyType":"msg","rules":[{"t":"cont","v":"0x00","vt":"str"},{"t":"else"}],"checkall":"true","repair":false,"outputs":2,"x":1430,"y":1640,"wires":[[],["7809fd85.767624"]]},{"id":"aa839120.af379","type":"ui_group","name":"Last Update","tab":"1862e313.45ffad","order":19,"disp":true,"width":"7","collapse":true},{"id":"7b626d8c.6f0df4","type":"zigbee2mqtt-server","name":"zigbee2mqtt","host":"localhost","mqtt_port":"1883","mqtt_username":"","mqtt_password":"","tls":"","usetls":false,"base_topic":"zigbee2mqtt"},{"id":"1862e313.45ffad","type":"ui_tab","name":"Mike","icon":"dashboard","order":2,"disabled":false,"hidden":false}]

There are more devices not shown (to the left), and the wires coming in from the top at the end of the flow enable me to delete rows, or clear the table data, and are not really relevant as they're triggered manually.

I'm not really wedded to this flow, if there is a better way to skin the cat.

[Steve-Mcl]

The issue is you rely on callbacks from the table to populate your flow variable tableData2

When the browser is closed, the ui_table doesnt run & therefore no callback.

A workaround would be to log data to the flow variable at server-side (not via a callback from client-side)

[GeorgeCastanza]

Thanks, any hints on how to go about doing that?

Edit: I know how to set flow variables etc, just not how to dynamically update parts of them when they arrive at various times.

[Christian-Me]

Perhaps this post give you a starting point:

Problem with data displayed in UI-Table when switching Tabs

So something like a logfile .... This is using a little bit more complex (but universal) subflow I use for all my tables. But you can do it more simple too: A) a function node which gets an array (of lines) from the flow context and pushes the new line at the end B) a second function node triggered by ui-control gets the same context array and send it to ui-table I splittend it up to different nodes to make it easier to understand / debug - The tricky part is to scroll the table if you in…

All depends on the type of data. The flow in the linked post is for growing data, like logging events. If you are updating data the tabledata.push() Is not enough. Then you need an object with an unique id for every row to keep your data in sync.

[GeorgeCastanza]

Thanks Christian-Me, I had a go, but it's beyond my paygrade. All I want to send is this (and display in a table):

{
   "command":"updateOrAddData",
   "arguments":[
       [
           {
               "id":msg.device.friendly_name,
               "value":msg.payload,
               "name":msg.device.friendly_name
            }
        ]
    ],
   "returnPromise":true
}

value is a timestamp set to ()now. YYYY-MM-DD HH:mm 2021-09-13 21:42

[Christian-Me]

Then the small flow in the linked post is nearly exactly what you are looking for. (if the timestamp value is unique and always progressing) as every new messages creates a new row and scrolls down)

Instead of the "demo data" node feed in your data as msg.payload into the "store data" node. to limit the max. amount of lines (see post below linked post) to be stored update the stopre data node (max could be easily around 1-5k if nessesary)

If you do not sort your table everything should be fine.

But as your date format should be save sorted alphabetically but if you like you can format the value / timestamp field as described here: Ui-table date sort - #4 by Christian-Me. (use an inject node to send msg.ui_control JSON together with an empty payload.