Here is an example of doing analytical analysis on complex tabular data with the help of the ObservableHQ Plot library and nodejs-polars.
nodejs-polars creates structured data for analytics in similar vein to Pandas in Python. In this example, converting text from a CSV file downloaded from the Internet and converted to a DataFrame for analysis.
Once the data has been queried/filtered, it is passed to Plot and turned into a line chart - within Node-RED.
That chart is then sent via a uibuilder uib-element node to a uibuilder node and the chart is rendered in the browser.
This may sound complex but it actually is pretty easy:
[{"id":"ea5770f642e51368","type":"uibuilder","z":"30fdd9a9702231b0","name":"","topic":"","url":"data-dash-eg","okToGo":true,"fwdInMessages":false,"allowScripts":false,"allowStyles":false,"copyIndex":true,"templateFolder":"blank","extTemplate":"","showfolder":false,"reload":false,"sourceFolder":"src","deployedVersion":"7.1.0","showMsgUib":false,"title":"","descr":"","editurl":"vscode://file/src/uibRoot/responsive-table-eg/?windowId=_blank","x":560,"y":6480,"wires":[["ff772a2310db69c0"],["d29760cb504c218b"]]},{"id":"ff772a2310db69c0","type":"debug","z":"30fdd9a9702231b0","name":"debug 2705","active":true,"tosidebar":true,"console":false,"tostatus":true,"complete":"true","targetType":"full","statusVal":"","statusType":"counter","x":755,"y":6460,"wires":[],"l":false},{"id":"d29760cb504c218b","type":"debug","z":"30fdd9a9702231b0","name":"debug 2706","active":false,"tosidebar":true,"console":false,"tostatus":true,"complete":"true","targetType":"full","statusVal":"","statusType":"counter","x":695,"y":6500,"wires":[],"l":false},{"id":"6b216267a6d2fe66","type":"uib-element","z":"30fdd9a9702231b0","name":"","topic":"","elementtype":"html","parent":"body","parentSource":"","parentSourceType":"str","elementid":"","elementId":"","elementIdSourceType":"str","heading":"","headingSourceType":"str","headingLevel":"h2","data":"payload","dataSourceType":"msg","position":"last","positionSourceType":"str","passthrough":false,"confData":{},"x":630,"y":6580,"wires":[["ace57c2e3bd5fa15"]]},{"id":"55baa258b1613613","type":"inject","z":"30fdd9a9702231b0","name":"","props":[{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"plot example","x":115,"y":6580,"wires":[["722af8b2a5b3e27a"]],"l":false},{"id":"722af8b2a5b3e27a","type":"http request","z":"30fdd9a9702231b0","name":"get sample csv","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://gist.githubusercontent.com/agustinustheo/195f32a4a6c68c493056c883d959ca35/raw/c7363d8b916ab00a2d1747adb89fca120da29f42/mock_financial_data.csv","tls":"","persist":false,"proxy":"","insecureHTTPParser":false,"authType":"","senderr":false,"headers":[],"x":240,"y":6580,"wires":[["363349c9c0871ef4"]]},{"id":"363349c9c0871ef4","type":"function","z":"30fdd9a9702231b0","name":"conv to df","func":"const SVGElement = ld.SVGElement\n\nconst df = pl.readCSV(msg.payload, { sep: \",\" });\nlet records = df.tail(40).toRecords();\n\nmsg.convertedArray = records.map(item => {\n return {\n NetIncome: item.NetIncome,\n Date: new Date(item.Date)\n };\n});\n\nconst document = new ld.DOMParser().parseFromString(\n `<!DOCTYPE html><html lang=\"en\"></html>`,\n \"text/html\",\n);\n\nconst plot = Plot.plot({\n x: { type: \"band\" },\n y: { grid: true },\n marks: [\n Plot.line(msg.convertedArray, { x: \"Date\", y: \"NetIncome\" }),\n ],\n document\n})\n\nmsg.payload = plot.outerHTML\n\n// msg.doc = document\n\nreturn msg;","outputs":1,"timeout":0,"noerr":0,"initialize":"","finalize":"","libs":[{"var":"pl","module":"nodejs-polars"},{"var":"ld","module":"linkedom"},{"var":"Plot","module":"@observablehq/plot"},{"var":"d3","module":"d3"}],"x":420,"y":6580,"wires":[["6b216267a6d2fe66"]]},{"id":"77cbd1d716645a2e","type":"link in","z":"30fdd9a9702231b0","name":"link in 78","links":["ace57c2e3bd5fa15"],"x":185,"y":6480,"wires":[["6da4651f84e62de3"]]},{"id":"ace57c2e3bd5fa15","type":"link out","z":"30fdd9a9702231b0","name":"link out 145","mode":"link","links":["77cbd1d716645a2e"],"x":785,"y":6580,"wires":[]},{"id":"6da4651f84e62de3","type":"uib-update","z":"30fdd9a9702231b0","name":"Update Page Heading","topic":"","mode":"update","modeSourceType":"update","cssSelector":"h1","cssSelectorType":"str","slotSourceProp":"Example: Plotting complex data using @observablehq/plot","slotSourcePropType":"str","attribsSource":"","attribsSourceType":"msg","slotPropMarkdown":false,"x":340,"y":6480,"wires":[["ea5770f642e51368"]]}]
Note that we add some helper modules to the function node.
LinkDom is needed to simulate the browser DOM and lets Plot create the chart offline.
Here is the function code:
const SVGElement = ld.SVGElement
const df = pl.readCSV(msg.payload, { sep: "," })
let records = df.tail(40).toRecords()
msg.convertedArray = records.map(item => {
return {
NetIncome: item.NetIncome,
Date: new Date(item.Date)
}
})
const document = new ld.DOMParser().parseFromString(
`<!DOCTYPE html><html lang="en"></html>`,
"text/html",
)
const plot = Plot.plot({
x: { type: "band" },
y: { grid: true },
marks: [
Plot.line(msg.convertedArray, { x: "Date", y: "NetIncome" }),
],
document
})
msg.payload = plot.outerHTML
return msg
Apart from tweaking the data processing, the only thing you need to otherwise change for your own charts is the data that produces the plot.
This is based on an example I found that was suggesting the use of a Jupyter notebook with Deno.
For extra fun, lets add a 2nd plot to the output, also based from that website. We just need to tweak the function node a bit:
const SVGElement = ld.SVGElement
const document = new ld.DOMParser().parseFromString(
`<!DOCTYPE html><html lang="en"></html>`,
"text/html",
)
const df = pl.readCSV(msg.payload, { sep: "," })
let records = df.tail(40).toRecords()
const convertedArray = records.map(item => {
return {
NetIncome: item.NetIncome,
Date: new Date(item.Date)
}
})
const plot = Plot.plot({
x: { type: "band" },
y: { grid: true },
marks: [
Plot.line(convertedArray, { x: "Date", y: "NetIncome" }),
],
document
})
const sampleData2 = []
const categories = ['Salaries', 'R&D', 'Marketing', 'Utilities', 'Rent', 'Equipment', 'Software', 'Hardware', 'Consulting', 'Office Supplies']
for (let i = 0; i < records.length; i++) {
const currentRecord = records[i]
for (let x = 0; x < categories.length; x++) {
const currentCategory = categories[x]
sampleData2.push({
date: new Date(currentRecord["Date"]),
category: currentCategory,
count: currentRecord[currentCategory],
})
}
}
const plot2 = Plot.plot({
x: { type: "band" },
y: { grid: true },
marks: [
Plot.barY(sampleData2, { x: "date", y: "count", fill: "category" }),
Plot.ruleY([0]),
],
color: { legend: true },
document,
})
msg.payload = plot.outerHTML
msg.payload += plot2.outerHTML
return msg
Everything else is the same.
Of course, you could do a lot of this in the front-end code instead, both approaches are equally valid, which is best for you depends on numbers of users, size of data, frequency of updates, etc.
Lets do a final tweak to get the outputs looking a bit nicer on the page:
msg.payload = `<article><h2>Plot 1: Net Income by Day</h2>${plot.outerHTML}</article>`
msg.payload += `<article><h2>Plot 2: Costs by Day</h2>${plot2.outerHTML}</article>`
