Include variables from external .js file

General
1

I have a PV logger, which creates js files with the data from the photovoltaic device.
I download these files with a FTP client within Node-RED (or an external script) on a regular basis.
How can I use / import / start these files and the variables in it in Node-RED to visualize and analyze them?

all files only with parts of their content:

base_vars.js:

var Serialnr = 1345982407
var Firmware = "2.8.3 Build 53"

days_hist.js:

da[dx++]="02.11.23|2321;632|2177;590|2106;1042|2174;977|1865;718"
da[dx++]="01.11.23|16497;4541|15495;4283|9120;3140|9149;3142|8286;2881"
da[dx++]="31.10.23|7954;2990|7480;2811|5367;1516|5464;1525|4911;1374"
da[dx++]="30.10.23|14182;4645|13347;4412|4923;1382|4992;1376|4512;1268"
da[dx++]="29.10.23|5809;3852|5464;3640|4229;3166|4382;3557|3938;3103"
da[dx++]="28.10.23|9912;4945|9275;4643|4106;1206|4157;1202|3751;1089"

years.js:

ye[yx++]="31.12.24|315857|298254|208025|208678|187386"
ye[yx++]="31.12.23|9557146|9004940|6212462|6290694|5654414"
ye[yx++]="31.12.22|11120779|10487135|7231538|7294839|6556857"
ye[yx++]="31.12.21|10095952|9493831|6616246|6664464|5984497"
2

Happy Cake day :grinning:

3

In theory, you could use require to evaluate the files, but that can be risky (unless you 100% trust the js files and they have no other dependencies (not shown in your excepts).

Personally, I would use regex to extract the data but without an explantion of the columns, it will be a pain to guess

There must be some variable declarations and other stuff in those files (above and below the da[dx++] = "data|data|..." lines that would provide hints.

PS happy cake day

4

Thanks!! I just realized that, while writing :grin:

5

According to https://stevesnoderedguide.com/using-nodejs-modules-in-node-red

grafik
grafik582Ă—108 2.16 KB

In the function node:

msg.payload =global.get("basejs");

Can I find the above single variables under msg.payload.xxx?
The following debug node does not show anything!?

While deploy the following errors shows:

function : (error)
"Error: Module not allowed"
6

That is for importing NPM modules.

7

If you really want to blindly require some external generated JS that you/we dont know what it does, then you can add require in settings.js (example) and then use const myRequire = global.get("require") then you can do const data = myRequire('/path/to/file') but i still dont recommend it.

Why not? - because I have no idea what that JS file does (you only showed a portion). For all I know, that js (attempts) to import other files, writes to filesystem, deletes files, corrupts prototypes, calls process.exit at the end or any number of things. Thats why I said:

8

In case you are wondering - it is not hard to do - its just you havent provided additional data from the js files so I have no idea what the meaning of the data is and therfore anything I create for you is pretty much guess work!

But hey ho, hear you go

chrome_dEtRzHGIpX

Demo Flow

[{"id":"66a8322784462377","type":"inject","z":"20165f674cef3579","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":700,"y":900,"wires":[["9ec7a64ed9b390d7"]]},{"id":"9ec7a64ed9b390d7","type":"file in","z":"20165f674cef3579","name":"","filename":"base_vars.js","filenameType":"str","format":"utf8","chunk":false,"sendError":false,"encoding":"none","allProps":false,"x":890,"y":900,"wires":[["56ef08dc3113e9d7"]]},{"id":"f5078eb304270ee0","type":"debug","z":"20165f674cef3579","name":"debug 15","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1320,"y":900,"wires":[]},{"id":"cf74b509e53ac901","type":"inject","z":"20165f674cef3579","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":780,"y":780,"wires":[["986f2813f8c0cceb"]]},{"id":"986f2813f8c0cceb","type":"template","z":"20165f674cef3579","name":"day_hist.js sample","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"// pure speculation on what might be in the file\n\nconst {saveToDB} = require('database.js')\n\nvar da = []\nvar dx = 0\nda[dx++]=\"02.11.23|2321;632|2177;590|2106;1042|2174;977|1865;718\"\nda[dx++]=\"01.11.23|16497;4541|15495;4283|9120;3140|9149;3142|8286;2881\"\nda[dx++]=\"31.10.23|7954;2990|7480;2811|5367;1516|5464;1525|4911;1374\"\nda[dx++]=\"30.10.23|14182;4645|13347;4412|4923;1382|4992;1376|4512;1268\"\nda[dx++]=\"29.10.23|5809;3852|5464;3640|4229;3166|4382;3557|3938;3103\"\nda[dx++]=\"28.10.23|9912;4945|9275;4643|4106;1206|4157;1202|3751;1089\"\n\n// export to database\nsaveToDB(da)","output":"str","x":970,"y":780,"wires":[["75ecdc1104e5ce74"]]},{"id":"75ecdc1104e5ce74","type":"file","z":"20165f674cef3579","name":"","filename":"days_hist.js","filenameType":"str","appendNewline":true,"createDir":false,"overwriteFile":"false","encoding":"none","x":1180,"y":780,"wires":[[]]},{"id":"3997cbfe05436c30","type":"inject","z":"20165f674cef3579","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":780,"y":820,"wires":[["ca0499bc33731f75"]]},{"id":"ca0499bc33731f75","type":"template","z":"20165f674cef3579","name":"years.js sample","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"// pure speculation on what might be in the file\n\n/*** years.js: ***/\nconst {saveToDB} = require('database.js')\n\nye[yx++]=\"31.12.24|315857|298254|208025|208678|187386\"\nye[yx++]=\"31.12.23|9557146|9004940|6212462|6290694|5654414\"\nye[yx++]=\"31.12.22|11120779|10487135|7231538|7294839|6556857\"\nye[yx++]=\"31.12.21|10095952|9493831|6616246|6664464|5984497\"\n\n\n// export to database\nsaveToDB(da)\n\n","output":"str","x":960,"y":820,"wires":[["89d9fd1de6ddb441"]]},{"id":"89d9fd1de6ddb441","type":"file","z":"20165f674cef3579","name":"","filename":"years.js","filenameType":"str","appendNewline":true,"createDir":false,"overwriteFile":"false","encoding":"none","x":1170,"y":820,"wires":[[]]},{"id":"653ab22d3587ca7f","type":"template","z":"20165f674cef3579","name":"base_vars.js sample","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"// pure speculation on what might be in the file\n/*** base_vars.js ***/\n\nvar Serialnr = 1345982407\nvar Firmware = \"2.8.3 Build 53\"\n\nexport default {\n  Serialnr,\n  Firmware\n}\n\n\n","output":"str","x":980,"y":740,"wires":[["028a11c26c9c1ad2"]]},{"id":"0d5025d845151480","type":"inject","z":"20165f674cef3579","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":780,"y":740,"wires":[["653ab22d3587ca7f"]]},{"id":"028a11c26c9c1ad2","type":"file","z":"20165f674cef3579","name":"","filename":"base_vars.js","filenameType":"str","appendNewline":true,"createDir":false,"overwriteFile":"false","encoding":"none","x":1190,"y":740,"wires":[[]]},{"id":"053465714f5b29e0","type":"file in","z":"20165f674cef3579","name":"","filename":"days_hist.js","filenameType":"str","format":"utf8","chunk":false,"sendError":false,"encoding":"none","allProps":false,"x":890,"y":960,"wires":[["e83d19a72967ea94"]]},{"id":"ffcc943885ee0e43","type":"debug","z":"20165f674cef3579","name":"debug 19","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1320,"y":960,"wires":[]},{"id":"7366a99f5f63ec18","type":"inject","z":"20165f674cef3579","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":700,"y":960,"wires":[["053465714f5b29e0"]]},{"id":"f34bd88162ffc9a5","type":"inject","z":"20165f674cef3579","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":700,"y":1020,"wires":[["1f5593de07ef1081"]]},{"id":"1f5593de07ef1081","type":"file in","z":"20165f674cef3579","name":"","filename":"years.js","filenameType":"str","format":"utf8","chunk":false,"sendError":false,"encoding":"none","allProps":false,"x":880,"y":1020,"wires":[["ccf0ca0dc7b154bf"]]},{"id":"ccf0ca0dc7b154bf","type":"function","z":"20165f674cef3579","name":"extract year data values","func":"function parseYearData(rawData, fieldNames) {\n    const lines = rawData\n        .split('\\n')\n        .map(line => line.trim())\n        .filter(line => line.startsWith('ye[yx++]'));\n\n    return lines.map(line => {\n        const dataPart = line.split('=').pop().trim().replace(/^\"|\"$/g, '');\n        const [rawDate, ...values] = dataPart.split('|');\n\n        const [day, month, year] = rawDate.split('.');\n        const date = `20${year}/${month}/${day}`; // convert to yyyy/mm/dd\n\n        const parsedValues = [date, ...values.map(Number)];\n\n        return fieldNames.reduce((obj, key, i) => {\n            obj[key] = parsedValues[i];\n            return obj;\n        }, {});\n    });\n}\n\nconst fieldNames = [\"date\", \"ah\", \"tot\", \"dunno\", \"xxx\", \"yyy\"]\n\nmsg.payload = parseYearData(msg.payload, fieldNames);\n\nreturn msg;","outputs":1,"timeout":0,"noerr":0,"initialize":"","finalize":"","libs":[],"x":1110,"y":1020,"wires":[["387b2e308d60a689"]]},{"id":"387b2e308d60a689","type":"debug","z":"20165f674cef3579","name":"debug 20","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1320,"y":1020,"wires":[]},{"id":"56ef08dc3113e9d7","type":"function","z":"20165f674cef3579","name":"extract meta data values","func":"function extractMetaInfo(rawString) {\n    const result = {};\n\n    const serialMatch = rawString.match(/var\\s+Serialnr\\s*=\\s*(\\d+)/);\n    const firmwareMatch = rawString.match(/var\\s+Firmware\\s*=\\s*\"([^\"]+)\"/);\n\n    if (serialMatch) result.Serialnr = parseInt(serialMatch[1], 10);\n    if (firmwareMatch) result.Firmware = firmwareMatch[1];\n\n    return result;\n}\nmsg.payload = extractMetaInfo(msg.payload);\n\nreturn msg;","outputs":1,"timeout":0,"noerr":0,"initialize":"","finalize":"","libs":[],"x":1110,"y":900,"wires":[["f5078eb304270ee0"]]},{"id":"e83d19a72967ea94","type":"function","z":"20165f674cef3579","name":"extract day hist values","func":"function dataHistoryData(rawData, fieldNames) {\n    const lines = rawData\n        .split('\\n')\n        .map(line => line.trim())\n        .filter(line => line && line.includes('|'));\n\n    return lines.map(line => {\n        // Extract the right-hand side after the \"=\"\n        const dataPart = line.split('=').pop().trim().replace(/^\"|\"$/g, '');\n        const [rawDate, ...pairs] = dataPart.split('|');\n        const [day, month, year] = rawDate.split('.');\n        const date = `20${year}/${month}/${day}`;\n\n        const flatValues = pairs.flatMap(pair => pair.split(';').map(Number));\n        const values = [date, ...flatValues];\n\n        return fieldNames.reduce((obj, key, idx) => {\n            obj[key] = values[idx];\n            return obj;\n        }, {});\n    });\n}\n\nconst fieldNames = [\"date\", \"v1\", \"v2\", \"a1\", \"a2\", \"p1\", \"p2\", \"va1\", \"va2\"];\n\nmsg.payload = dataHistoryData(msg.payload, fieldNames);\n\nreturn msg;","outputs":1,"timeout":0,"noerr":0,"initialize":"","finalize":"","libs":[],"x":1100,"y":960,"wires":[["ffcc943885ee0e43"]]},{"id":"9dfa4a9a4782c18c","type":"comment","z":"20165f674cef3579","name":"write made up versions of your data to file","info":"","x":880,"y":700,"wires":[]},{"id":"68f6590ce63631fb","type":"comment","z":"20165f674cef3579","name":"read data from files and parse out the values","info":"","x":990,"y":860,"wires":[]}]
1 Like
9

I checked the current files. They contain only variables like in the example above, but I would NEVER download and use new files without proving, that they do not contain harmful code.
I'm going to test your code now. Thank you in advance!

10

Wow!

I read the code and could change a few things (for example, in "extract day hist values", the 5th data pair was missing), but would never ever have build this code myself!!!

BTW: There are 5 inverters, so the data pairs in "day hist" and the 5 numbers in years correspond to the 5 inverters.

11

No, but the logger has a display, where I can see different values for day, week and year, so I can compare these values with the data from the .js files.

Thanks again for your help!!

12

Another approach, at least for the base_vars.js, which contains more than 50 entries:
(in your example I have to set two lines for every entry)

grep for "=" (and so delete the other lines with possible code)
grep for not containing a "var" in the line
grep for not containing an "array" (there some array definitions, which I don't need)
sed "s/var //g"

This file can be injected in the INI node.

13

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.