D3.js assistance

Dashboard
1

Hello all,

First of all: I would like thank everyone involved in creating and supporting this programing tool!

I have a couple of notes and a question in regards to D3.js and node-red integration using DASHBOARD:

  1. Most of the time I get the D3.js charts to display whenever I "DEPLOY:FULL" (but not always).
  2. I almost never get the charts to show whenever "DEPOLY:MODIFIED_NODES" is used... I still don't understand why it does show as it has happened while refreshing the screen, or doing "DEPOLY:MODIFIED_NODES" once more.
  3. Is there something within the code I am failing to do correctly, so the charts get to come up right away?
  4. And the d3.js script be local to the computer so it loads faster?

NOTE:

  • An experienced UI friend of mine and I slightly modified and example I found on this forum that me greatly assisted me in getting an application started with d3.js and node-red. I believe it was originally created by @Andrei, and it can be found here: 3D plot in NodeRed

    [{"id":"b8ad1d13.b43a","type":"debug","z":"9bffbe5c.ccd4d","name":"","active":true,"tosidebar":true,"console":true,"tostatus":false,"complete":"payload","x":1040,"y":180,"wires":[]},{"id":"b0fe2a06.f6f0e8","type":"inject","z":"9bffbe5c.ccd4d","name":"","topic":"PUSH_BOTTON_1","payload":"welcome","payloadType":"str","repeat":"","crontab":"","once":false,"onceDelay":"","x":560,"y":40,"wires":[["2210b655.64f9ca"]]},{"id":"2210b655.64f9ca","type":"ui_template","z":"9bffbe5c.ccd4d","group":"36bda78d.d01f18","name":"testing payload","order":1,"width":"10","height":"12","format":"<!DOCTYPE html>\n<html>\n \n <head>\n <title>Scatter Plot</title>\n <script src=\"https://d3js.org/d3.v4.min.js\"></script>\n <style>\n .axis {\n fill: none;\n stroke: black;\n shape-rendering: crispEdges;\n }\n </style>\n \n </head>\n \n <body>\n \n <div id=\"plot\"></div>\n <div id=\"ELEMENT\"></div>\n <h2>{{msg.topic}}</h2>\n <h2>{{msg.payload}}</h2>\n \n <script>\n\n (function(scope) {\n \n // Watch for messages being send to this template node\n scope.$watch('msg', function (msg) {\n \n if (msg) {\n \n //var ELEMENT = document.getElementById('ELEMENT');\n ELEMENT = msg.payload;\n plotDraw();\n\n }\n \n });\n \n var w = 500, h = 500, pad = 50;\n \n var svg = d3.select(\"#plot\")\n .append(\"svg\")\n .attr(\"height\", h)\n .attr(\"width\", w);\n \n var xAxisGroup = svg.append(\"g\")\n .attr(\"class\", \"axis\")\n .attr(\"transform\", \"translate(0,\" + (h - pad) + \")\");\n \n var yAxisGroup = svg.append(\"g\")\n .attr(\"class\", \"axis\")\n .attr(\"transform\", \"translate(\" + pad +\", 0)\");\n \n const plotDraw = function(){\n var dataset = [ ELEMENT , [20, 20, 20],[30, 30, 30], [40, 40, 40], [50, 50, 50], [60, 60, 60]];\n \n var xScale = d3.scaleLinear()\n .domain([0, d3.max(dataset, function(d) { return d[0]; })])\n .range([pad, w - pad]);\n \n var yScale = d3.scaleLinear()\n .domain([0, d3.max(dataset, function(d) { return d[1]; })])\n .range([h - pad, pad]);\n \n var rScale = d3.scaleLinear()\n .domain([0, d3.max(dataset, function(d) { return d[2]; })])\n .range([1, 30]);\n \n var xAxis = d3.axisBottom(xScale);\n var yAxis = d3.axisLeft(yScale);\n \n var circ = svg.selectAll(\"circle\")\n .data(dataset);\n \n var circEnter = circ.enter()\n .append(\"circle\");\n \n circEnter\n .attr(\"fill\", \"red\")\n .attr(\"opacity\", 0.5);\n \n circ.merge(circEnter)\n .attr(\"cx\", function(d) { return xScale(d[0]); })\n .attr(\"cy\", function(d) { return yScale(d[1]); })\n .attr(\"r\", function(d) { return rScale(d[2]); })\n \n xAxisGroup.call(xAxis);\n yAxisGroup.call(yAxis);\n\n };\n \n plotDraw(); \n \n })(scope);\n\n </script>\n </body>\n</html>","storeOutMessages":true,"fwdInMessages":true,"templateScope":"local","x":840,"y":180,"wires":[["b8ad1d13.b43a"]]},{"id":"be0e7a5a.035068","type":"inject","z":"9bffbe5c.ccd4d","name":"","topic":"PUSH_BOTTON_2","payload":"not welcome","payloadType":"str","repeat":"","crontab":"","once":false,"onceDelay":"","x":550,"y":180,"wires":[["2210b655.64f9ca"]]},{"id":"87a9ec37.e7324","type":"ui_slider","z":"9bffbe5c.ccd4d","name":"","label":"X","tooltip":"","group":"36bda78d.d01f18","order":2,"width":0,"height":0,"passthru":true,"outs":"all","topic":"X","min":0,"max":"100","step":1,"x":450,"y":280,"wires":[["f2b66220.78ab6"]]},{"id":"93f55155.11c24","type":"ui_slider","z":"9bffbe5c.ccd4d","name":"","label":"Y","tooltip":"","group":"36bda78d.d01f18","order":3,"width":0,"height":0,"passthru":true,"outs":"all","topic":"Y","min":0,"max":"100","step":1,"x":450,"y":340,"wires":[["f2b66220.78ab6"]]},{"id":"7176f2ca.e3144c","type":"ui_slider","z":"9bffbe5c.ccd4d","name":"","label":"R","tooltip":"","group":"36bda78d.d01f18","order":4,"width":0,"height":0,"passthru":true,"outs":"all","topic":"R","min":0,"max":"100","step":1,"x":450,"y":400,"wires":[["f2b66220.78ab6"]]},{"id":"f2b66220.78ab6","type":"function","z":"9bffbe5c.ccd4d","name":"","func":"var X = {};\nvar Y = {};\nvar R = {};\nvar ELEMENT = {};\nvar STATE = \"\";\n\nif (msg.topic == \"X\") {\n\tcontext.X = msg.payload;\n\tSTATE = 'X';\n}\n\nif (msg.topic == \"Y\") {\n\tcontext.Y = msg.payload;\n STATE = 'Y';\n}\n\nif (msg.topic == \"R\") {\n\tcontext.R = msg.payload;\n STATE = 'R';\n}\n\nELEMENT.payload = [context.X || 0, context.Y || 0, context.R || 0];\nELEMENT.topic = 'ELEMENT: '+STATE;\n\nreturn ELEMENT;","outputs":1,"noerr":0,"x":630,"y":340,"wires":[["fb5a87ca.dc5768","2210b655.64f9ca"]]},{"id":"fb5a87ca.dc5768","type":"debug","z":"9bffbe5c.ccd4d","name":"","active":false,"tosidebar":true,"console":true,"tostatus":false,"complete":"payload","x":1040,"y":340,"wires":[]},{"id":"5295a89d.50f1c8","type":"inject","z":"9bffbe5c.ccd4d","name":"","topic":"X","payload":"10","payloadType":"num","repeat":"","crontab":"","once":true,"onceDelay":".1","x":110,"y":340,"wires":[["d35c4de.f8c17b"]]},{"id":"d35c4de.f8c17b","type":"ui_button","z":"9bffbe5c.ccd4d","name":"","group":"36bda78d.d01f18","order":4,"width":0,"height":0,"passthru":true,"label":"*** RESET ***","tooltip":"","color":"","bgcolor":"","icon":"","payload":"10","payloadType":"num","topic":"","x":250,"y":340,"wires":[["87a9ec37.e7324","93f55155.11c24","7176f2ca.e3144c"]]},{"id":"36bda78d.d01f18","type":"ui_group","z":"","name":"D3.js","tab":"4b1c9046.2401e","order":2,"disp":true,"width":"10","collapse":false},{"id":"4b1c9046.2401e","type":"ui_tab","z":"9bffbe5c.ccd4d","name":"D3.js","icon":"dashboard","order":9}]

(I can't yet figure out how to "one-line this code"... apologize me for the eyesore)

2

Hi , indeed the code can be refactored to avoid the "refreshing issue" at the same time making it simpler.

At the end of the day what you want is that your circle follows the moving of the sliders, right ?

You can achieve that without using context and without using the function node. More importantly, the new code should handle better the initialization (which is, in my opinion, the weak point in the code you posted).

Below code is an attempt to address those points. Check if it works for you. I created a new dashboard group to ease the visualization but of course, you can change it back to your preference.

R-01
R-011138Ă—544 44.5 KB

[{"id":"aeed00d2.47e62","type":"tab","label":"Flow 4","disabled":false,"info":""},{"id":"eec8e8c3.3f76a8","type":"debug","z":"aeed00d2.47e62","name":"","active":false,"tosidebar":false,"console":false,"tostatus":true,"complete":"payload","targetType":"msg","x":930,"y":340,"wires":[]},{"id":"4014817.fc5648","type":"inject","z":"aeed00d2.47e62","name":"","topic":"PUSH_BOTTON_1","payload":"welcome","payloadType":"str","repeat":"","crontab":"","once":false,"onceDelay":"","x":560,"y":200,"wires":[["8a53a4c1.aa8f38"]]},{"id":"8a53a4c1.aa8f38","type":"ui_template","z":"aeed00d2.47e62","group":"d1cee596.acf6e8","name":"testing payload","order":1,"width":"12","height":"12","format":"<!DOCTYPE html>\n<html>\n \n <head>\n <title>Scatter Plot</title>\n <script src=\"https://d3js.org/d3.v4.min.js\"></script>\n <style>\n .axis {\n fill: none;\n stroke: black;\n shape-rendering: crispEdges;\n }\n </style>\n \n </head>\n \n <body>\n \n <div id=\"plot\"></div>\n <div id=\"ELEMENT\"></div>\n <h2>{{msg.topic}}</h2>\n <h2>{{msg.payload}}</h2>\n \n <script>\n\n (function(scope) {\n \n // Watch for messages being send to this template node\n scope.$watch('msg', function (msg) {\n \n if (msg) {\n \n //var ELEMENT = document.getElementById('ELEMENT');\n ELEMENT = msg.payload;\n plotDraw();\n\n }\n \n });\n \n var w = 500, h = 500, pad = 50;\n \n var svg = d3.select(\"#plot\")\n .append(\"svg\")\n .attr(\"height\", h)\n .attr(\"width\", w);\n \n var xAxisGroup = svg.append(\"g\")\n .attr(\"class\", \"axis\")\n .attr(\"transform\", \"translate(0,\" + (h - pad) + \")\");\n \n var yAxisGroup = svg.append(\"g\")\n .attr(\"class\", \"axis\")\n .attr(\"transform\", \"translate(\" + pad +\", 0)\");\n \n const plotDraw = function(){\n var dataset = [ ELEMENT , [20, 20, 20],[30, 30, 30], [40, 40, 40], [50, 50, 50], [60, 60, 60]];\n \n var xScale = d3.scaleLinear()\n .domain([0, d3.max(dataset, function(d) { return d[0]; })])\n .range([pad, w - pad]);\n \n var yScale = d3.scaleLinear()\n .domain([0, d3.max(dataset, function(d) { return d[1]; })])\n .range([h - pad, pad]);\n \n var rScale = d3.scaleLinear()\n .domain([0, d3.max(dataset, function(d) { return d[2]; })])\n .range([1, 30]);\n \n var xAxis = d3.axisBottom(xScale);\n var yAxis = d3.axisLeft(yScale);\n \n var circ = svg.selectAll(\"circle\")\n .data(dataset);\n \n var circEnter = circ.enter()\n .append(\"circle\");\n \n circEnter\n .attr(\"fill\", \"red\")\n .attr(\"opacity\", 0.5);\n \n circ.merge(circEnter)\n .attr(\"cx\", function(d) { return xScale(d[0]); })\n .attr(\"cy\", function(d) { return yScale(d[1]); })\n .attr(\"r\", function(d) { return rScale(d[2]); })\n \n xAxisGroup.call(xAxis);\n yAxisGroup.call(yAxis);\n\n };\n \n plotDraw(); \n \n })(scope);\n\n </script>\n </body>\n</html>","storeOutMessages":true,"fwdInMessages":true,"templateScope":"local","x":940,"y":260,"wires":[[]]},{"id":"25ce775f.cfb328","type":"inject","z":"aeed00d2.47e62","name":"","topic":"PUSH_BOTTON_2","payload":"not welcome","payloadType":"str","repeat":"","crontab":"","once":false,"onceDelay":"","x":550,"y":260,"wires":[["8a53a4c1.aa8f38"]]},{"id":"4df33827.c98788","type":"ui_slider","z":"aeed00d2.47e62","name":"","label":"X","tooltip":"","group":"30dba676.df740a","order":1,"width":0,"height":0,"passthru":true,"outs":"all","topic":"X","min":0,"max":"100","step":1,"x":410,"y":340,"wires":[["b94df073.f6dee"]]},{"id":"d48fccb.ddc673","type":"ui_slider","z":"aeed00d2.47e62","name":"","label":"Y","tooltip":"","group":"30dba676.df740a","order":2,"width":0,"height":0,"passthru":true,"outs":"all","topic":"Y","min":0,"max":"100","step":1,"x":410,"y":400,"wires":[["b94df073.f6dee"]]},{"id":"d2d5ed25.093d3","type":"ui_slider","z":"aeed00d2.47e62","name":"","label":"R","tooltip":"","group":"30dba676.df740a","order":3,"width":0,"height":0,"passthru":true,"outs":"all","topic":"R","min":0,"max":"100","step":1,"x":410,"y":460,"wires":[["b94df073.f6dee"]]},{"id":"c1a8e70f.b41a38","type":"debug","z":"aeed00d2.47e62","name":"","active":true,"tosidebar":false,"console":false,"tostatus":true,"complete":"payload","targetType":"msg","x":830,"y":400,"wires":[]},{"id":"cb1c73da.06807","type":"ui_button","z":"aeed00d2.47e62","name":"","group":"30dba676.df740a","order":4,"width":0,"height":0,"passthru":true,"label":"*** RESET ***","tooltip":"","color":"","bgcolor":"","icon":"","payload":"10","payloadType":"num","topic":"","x":300,"y":160,"wires":[["55d09af4.6a8474"]]},{"id":"4451eece.f674e","type":"inject","z":"aeed00d2.47e62","name":"","topic":"","payload":"","payloadType":"date","repeat":"","crontab":"","once":true,"onceDelay":0.1,"x":310,"y":80,"wires":[["adf9445e.b8fc28"]]},{"id":"adf9445e.b8fc28","type":"change","z":"aeed00d2.47e62","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"[10,10,10]","tot":"json"},{"t":"set","p":"topic","pt":"msg","to":"INIT","tot":"str"},{"t":"set","p":"x","pt":"msg","to":"5","tot":"num"},{"t":"set","p":"y","pt":"msg","to":"5","tot":"num"},{"t":"set","p":"r","pt":"msg","to":"5","tot":"num"}],"action":"","property":"","from":"","to":"","reg":false,"x":580,"y":80,"wires":[["4b621e24.399d3"]]},{"id":"8eaacd8a.8dca7","type":"change","z":"aeed00d2.47e62","name":"X","rules":[{"t":"set","p":"payload","pt":"msg","to":"x","tot":"msg"},{"t":"set","p":"topic","pt":"msg","to":"X","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":250,"y":340,"wires":[["4df33827.c98788"]]},{"id":"4b621e24.399d3","type":"link out","z":"aeed00d2.47e62","name":"","links":["bfba415b.41e41"],"x":795,"y":100,"wires":[]},{"id":"1eea6c49.5a2f44","type":"change","z":"aeed00d2.47e62","name":"Y","rules":[{"t":"set","p":"payload","pt":"msg","to":"y","tot":"msg"},{"t":"set","p":"topic","pt":"msg","to":"Y","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":250,"y":400,"wires":[["d48fccb.ddc673"]]},{"id":"25aa325b.ca6c5e","type":"change","z":"aeed00d2.47e62","name":"R","rules":[{"t":"set","p":"payload","pt":"msg","to":"r","tot":"msg"},{"t":"set","p":"topic","pt":"msg","to":"R","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":250,"y":460,"wires":[["d2d5ed25.093d3"]]},{"id":"bfba415b.41e41","type":"link in","z":"aeed00d2.47e62","name":"","links":["4b621e24.399d3"],"x":100,"y":400,"wires":[["1eea6c49.5a2f44","8eaacd8a.8dca7","25aa325b.ca6c5e"]]},{"id":"b94df073.f6dee","type":"join","z":"aeed00d2.47e62","name":"","mode":"custom","build":"object","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":true,"timeout":"","count":"1","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":570,"y":400,"wires":[["c1a8e70f.b41a38","6a16e75c.511898"]]},{"id":"6a16e75c.511898","type":"change","z":"aeed00d2.47e62","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"[msg.payload.X, msg.payload.Y, msg.payload.R]","tot":"jsonata"}],"action":"","property":"","from":"","to":"","reg":false,"x":700,"y":340,"wires":[["8a53a4c1.aa8f38","eec8e8c3.3f76a8"]]},{"id":"55d09af4.6a8474","type":"change","z":"aeed00d2.47e62","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"[10,10,10]","tot":"json"},{"t":"set","p":"topic","pt":"msg","to":"INIT","tot":"str"},{"t":"set","p":"x","pt":"msg","to":"10","tot":"num"},{"t":"set","p":"y","pt":"msg","to":"10","tot":"num"},{"t":"set","p":"r","pt":"msg","to":"10","tot":"num"}],"action":"","property":"","from":"","to":"","reg":false,"x":580,"y":120,"wires":[["4b621e24.399d3"]]},{"id":"76157e86.f56e5","type":"inject","z":"aeed00d2.47e62","name":"","topic":"","payload":"","payloadType":"date","repeat":"","crontab":"","once":true,"onceDelay":"0.5","x":310,"y":120,"wires":[["55d09af4.6a8474"]]},{"id":"d1cee596.acf6e8","type":"ui_group","z":"","name":"G1","tab":"22f9baf9.0afad6","disp":true,"width":"12","collapse":false},{"id":"30dba676.df740a","type":"ui_group","z":"","name":"G2","tab":"22f9baf9.0afad6","disp":true,"width":"6","collapse":false},{"id":"22f9baf9.0afad6","type":"ui_tab","z":"","name":"T1","icon":"dashboard","disabled":false,"hidden":false}]
3

I missed this part of your question.

I like the approach that was explained here:

In order to use external js libs from inside your ui_template code, it's best to put the src links and the html/css/svg code into a separate templates. In the first one, choose the " Add to site <head> section " option, and include your library references:

At the same time, you can load the d3 library to a static folder in your Node-RED server. You will need to set up the static file in ´settings.js` (if not done yet).
In my setup I have this static folder:

29 Jan 10:24:20 - [info] HTTP Static : C:\Users\OCM.node-red\static

Inside this folder I have one folder for each version of d3:

r-01

I use a script line like the one below to load the d3 library from the local folder:

<script src="/d3v4/d3.min.js"></script>

Note: if you use d3 version 3 then there is no need to load as Node-RED uses this library in its core. It is loaded automatically (someone please correct me if I am wrong).

4

@Andrei

Thank you kindly for the vast information provided!

*** First Response Post:

  • Use/abuse of context usage: You are correct here... I am actually learning its actual purpose. I am using it as "constant" that lives within the flow and gets updated only once needed it to... but it looks clearer and cleaner to use the slider control setup as you had it layout.

  • Initialization process: I am also learning d3.js ... and believe me this one was very much copy/paste plus 10~15 minutes from my buddy giving me a hand to get it running... It does makes sense what you are doing, but question here: Why inject/initialize the scripts 2 consecutive times, using 0.1 seconds apart, and with different values between them?... I can understand the reset setting but kind of lost me on the init back-to-back approach.

  • Also, honest question: are modules like split/join/switch more efficient that functions doing the same? I like using either one, but I figure functions can save "space" and node usage (example in FRED)... I could definitely be wrong there.

  • and with your code, I am still seeing the same behavior that I had previously encounter before: However I DEPLOY the code, the X/Y charts don't plot right away, I have to move something to force NR to enable the code to be re-DEPLOYable and it is then when the X/Y Charts populate. I figure maybe I should provide my current hardware configuration as it "could" be the culprit:

mac-mini, 2.6 Ghz i5, 8Gb Ram, Osx Mojave 10.14.2
Node-RED version: v0.19.5
Node.js version: v10.15.1
Darwin 18.2.0 x64 LE
Dashboard version 2.13.2 started at /ui

*** Second Response Post:

  • This is very valuable information and I will be including it as you mentioned into the code / appropriate files, and testing it during the rest of the week. I will update this post as needed.... now that I think about it: This could well be the reason the X/Y charts are not loading... time to check for sure.

*** ---- Note ----
So you know the main reason I am learning d3.js (and node-red for that matter) is because I want to make "Box Plots" to track delta/percentage data with somewhat "good" granularity, in very long period of times, which in my project should make data more efficiently visible and clearer to extract that plotting charts.... and I feel it will definitely should consume much less resources.

Anyhow: This will take some time but I believe it will be worth it in the long run, for me and others!

Again, Thanks!

5

Hi @ExpZeros, great feedback. I have a number of insights and considerations to share and discuss with you but I have to leave now. I will return tomorrow for this fruitful discussion. See you.

6

The reason for this "double injection" boils down to a couple of reasons: the fact that it is necessary to initialize X, Y, and R to the value 10 (it could be any other value though) and the way the dashboard slider node works.

For the initialization to be consistent we want the "moving circle" being plotted automatically at the initialization time and be in sync with the handles of the sliders (we don't want for instance the slider on position x, y, r = 10,10, 10 but the circle plotted with radius 0 and in the position x=10, y=0). Another reason to force the plotting of the circle at startup is to avoid any possible lack of sync due to browser caching. In order to force the plot all that is needed is to ensure that a message arrives at the template node at the startup. This will not happen if we inject the initial values only once and the reason is the way the slider node works. The slider node will only change its output when the input value changes. One way to force this change of values is injecting different values in sequence (but could use other artifacts (or artefacts, never know the correct word) , like using the trigger node). I hope this makes sense. I don't discard that it may be possible to find another way to force the initial plotting (it is amazing how many solutions we can find for the same problem when using Node-RED), but this was the first thing that came to my mind.

7

It is always a matter of personal preference. I tend to abuse of function nodes when I write a flow for my own usage, otherwise, my preference is to use whatever combination (core nodes, JavaScript code, contrib nodes, JSONata) will make the flow easier to understand after a period of several months.

2 Likes
8

Totally agree. Efficiency is almost never the issue, and readability almost always is.

Another boost to readability and maintainability may arrive when subflows get the ability to have properties defined on a per-instance basis, report status to the editor and the status node, and read and write global variables. This would let us define a segment of flow that does a clearly defined job and reuse it more flexibly than we can now -- something like a function node composed of other nodes. I suspect I will be using subflows much more in the future.

1 Like
9

@Andrei

  • Response to Initialization: Thanks for the clarification. It does make sense to "seeding" the d3.js library this way.

  • d3.js using local libraries: I was not able to work on this today, but very much on my list. I want to get this as iron down as I can before I start the "Box Plots" journey.

@Andrei @drmibell

  • Response to "JS Functions" || "Function Nodes": Thanks both of you for justifying this point. I agree on making code understandable and maintainable thus using "Function Nodes" does that the second one opens/imports code into NR. I will consider this next code builds!