When connecting to an FTP server we get a list of files that are ready for download.
When trying to get these we are only ever getting 1 file downloaded. If we wire the last function to the Debug node we get each file listed, but when wiring in the ftp node to download, debug reports the same file details and we only get the 1st file
Any one have any tips to get the all the files down
ftpFlow.json (7.7 KB)
`
Which FTP node are you using? node-red-contrib-???? possibly.
Create a simple flow with Inject nodes, the ftp node and a debug node that exhibits the problem you are seeing then select those nodes and export to the clipboard and paste them here. When pasting use the </> button at the top of the forum entry window.
Also tell us what hardware and OS you running on, what version of node-red and what version of nodejs (use node -v in a command window to find that.
Hi Colin
Got interrupted as i was editing the post and adding in the flow.
Its now attached the original
Also
Node.js ver 16.19.0
Node-red ver 3.0.2
OS Ubuntu Server 22.04 LTS
Node-Red installed to Native OS using
bash <(curl -sL https://raw.githubusercontent.com/node-red/linux-installers/master/deb/update-nodejs-and-nodered)
Cheers greg
You haven't said which ftp node.
Please paste the flow inline here
In order to make code readable and usable it is necessary to surround your code with three backticks (also known as a left quote or backquote ```)
```
code goes here
```
See this post for more details - How to share code or flow json
Also, that looks like a complex flow, is it one that I can just click nodes to see the problem, which is what I asked for?
Hi Colin
I can get the FTP node to work on its own fine, the problem is when adding the complexity of getting multiple files downloaded, so isolating the FTP node into a simple flow is not really viable.
The below flow gets a list of files from the FTP site, filters out the ones we are interested in, then attempts to download then in the node "Download Files" (highlighted in the image)
Flow embedded below
[
{
"id": "aee9c356815ed3fb",
"type": "inject",
"z": "df9b50497051c573",
"name": "Download NB PO",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
},
{
"p": "fcount",
"v": "5",
"vt": "flow"
}
],
"repeat": "",
"crontab": "",
"once": false,
"onceDelay": 0.1,
"topic": "",
"payload": "",
"payloadType": "date",
"x": 80,
"y": 80,
"wires": [
[
"8d2899ab05df840f"
]
]
},
{
"id": "8d2899ab05df840f",
"type": "advanced-ftp",
"z": "df9b50497051c573",
"ftp": "a94914e317f79762",
"operation": "list",
"dataType": "binary",
"filename": "",
"localFilename": "",
"workingDir": "/outbound",
"oldPath": "",
"newPath": "",
"command": "",
"recursive": false,
"useCompression": false,
"throwError": false,
"showError": true,
"name": "List files in Outbound",
"x": 160,
"y": 140,
"wires": [
[
"32174f913b56d83c"
]
]
},
{
"id": "32174f913b56d83c",
"type": "function",
"z": "df9b50497051c573",
"name": "init flow variables",
"func": "var count = flow.get('count') || 0;\nvar files = flow.get('files') || 0;\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 330,
"y": 180,
"wires": [
[
"239a73431c30c19f"
]
]
},
{
"id": "239a73431c30c19f",
"type": "split",
"z": "df9b50497051c573",
"name": "Splt to individual files",
"splt": "\\n",
"spltType": "str",
"arraySplt": 1,
"arraySpltType": "len",
"stream": false,
"addname": "",
"x": 500,
"y": 220,
"wires": [
[
"203566a67c35f5f3"
]
]
},
{
"id": "203566a67c35f5f3",
"type": "switch",
"z": "df9b50497051c573",
"name": "Select File Type",
"property": "payload.name",
"propertyType": "msg",
"rules": [
{
"t": "cont",
"v": "NRNZ_NZ_HL_PO",
"vt": "str"
},
{
"t": "nempty"
}
],
"checkall": "false",
"repair": false,
"outputs": 2,
"x": 560,
"y": 280,
"wires": [
[
"192b29a564073fb3"
],
[
"191fdb4e1b52a411"
]
]
},
{
"id": "192b29a564073fb3",
"type": "function",
"z": "df9b50497051c573",
"name": "Add Filename to array",
"func": "msg.payload = msg.payload.name;\n\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 720,
"y": 240,
"wires": [
[
"4e4ec4075d149f11"
]
]
},
{
"id": "191fdb4e1b52a411",
"type": "function",
"z": "df9b50497051c573",
"name": "Add NA to array",
"func": "msg.payload='NA';\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 700,
"y": 320,
"wires": [
[
"4e4ec4075d149f11"
]
]
},
{
"id": "4e4ec4075d149f11",
"type": "join",
"z": "df9b50497051c573",
"name": "Rejoin to Array",
"mode": "auto",
"build": "object",
"property": "payload",
"propertyType": "msg",
"key": "topic",
"joiner": "\\n",
"joinerType": "str",
"accumulate": true,
"timeout": "",
"count": "",
"reduceRight": false,
"reduceExp": "",
"reduceInit": "",
"reduceInitType": "",
"reduceFixup": "",
"x": 880,
"y": 280,
"wires": [
[
"9858f9790b4c18d8"
]
]
},
{
"id": "9858f9790b4c18d8",
"type": "function",
"z": "df9b50497051c573",
"name": "Filter out NA's from the array",
"func": "var filtArr = msg.payload;\n\nconst retArr = filtArr.filter(filterFiles)\n\n// let retObj = [];\n\n// for (let i = 0; i < retArr.length; i++) {\n// //obj[i] = arr[i];\n// let srcfile = '/outbound/' + retArr[i];\n// let destfile = '/media/share/sftpSAPTest/NB-SAP/Inbound/Staging' + retArr[i];\n// let cc = {'src':srcfile, 'dest':destfile};\n// retObj.push(cc);\n// }\n\n// msg.payload = retObj;\nmsg.payload = retArr;\n\n\nreturn msg;\n\n\nconst arrayToObject = (arr) => {\n let obj = {};\n for (let i = 0; i < arr.length; i++) {\n obj[i] = arr[i];\n }\n return obj;\n};\n\n\nfunction filterFiles(fname){\n return fname != 'NA';\n}",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 940,
"y": 340,
"wires": [
[
"c62ca996c0a4446d"
]
]
},
{
"id": "c62ca996c0a4446d",
"type": "split",
"z": "df9b50497051c573",
"name": "Seperate Array to download",
"splt": "\\n",
"spltType": "str",
"arraySplt": 1,
"arraySpltType": "len",
"stream": false,
"addname": "",
"x": 940,
"y": 400,
"wires": [
[
"25cc2c197580df6c"
]
]
},
{
"id": "25cc2c197580df6c",
"type": "function",
"z": "df9b50497051c573",
"name": "Set filename and localfilename",
"func": "msg.operation = \"get\";\n\nlet lclFolder = '/media/share/sftpSAPTest/NB-SAP/Inbound/Staging/';\nlet srcfolder = '/outbound/';\nmsg.localFilename = lclFolder + msg.payload;\nmsg.filename = srcfolder + msg.payload;\n\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 930,
"y": 460,
"wires": [
[
"1c4ef6b7ae988063"
]
]
},
{
"id": "1c4ef6b7ae988063",
"type": "ftp in",
"z": "df9b50497051c573",
"ftp": "abc2cb3d02ee077a",
"operation": "get",
"filename": "",
"localFilename": "",
"name": "Download Files",
"x": 920,
"y": 520,
"wires": [
[
"e3b27537257158c1"
]
]
},
{
"id": "e3b27537257158c1",
"type": "debug",
"z": "df9b50497051c573",
"name": "debug 11",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 880,
"y": 600,
"wires": []
},
{
"id": "a94914e317f79762",
"type": "advanced-ftp-config",
"host": "ftp.newellrubbermaid.com",
"port": "",
"secure": false,
"secureOptions": "",
"user": "xf00347",
"connTimeout": "",
"pasvTimeout": "",
"keepalive": "",
"name": "NB Test - Sales Orders"
},
{
"id": "abc2cb3d02ee077a",
"type": "ftp",
"host": "ftp.newellrubbermaid.com",
"port": "",
"secureOptions": "",
"user": "xf00347",
"connTimeout": "",
"pasvTimeout": "",
"keepalive": ""
}
]
You still have not told us which FTP node. There are 20 ftp nodes listed here: Library - Node-RED
Attach a Debug node to the output of 'Set filename and localfilename' and show us what it gives.
Seems a bit complicated splitting, filtering joining then splitting again, when you can use the switch node to reconstruct message sequence when removing array elements.
You should be able to split filter and send to ftp using one split node.
e.g.
[{"id":"aee9c356815ed3fb","type":"inject","z":"b9860b4b9de8c8da","name":"Download NB PO","props":[{"p":"payload"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[{\"name\":\"one\"},{\"name\":\"NRNZ_NZ_HL_PO\"}]","payloadType":"json","x":580,"y":480,"wires":[["239a73431c30c19f"]]},{"id":"239a73431c30c19f","type":"split","z":"b9860b4b9de8c8da","name":"Splt to individual files","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":800,"y":480,"wires":[["203566a67c35f5f3"]]},{"id":"203566a67c35f5f3","type":"switch","z":"b9860b4b9de8c8da","name":"Select File Type","property":"payload.name","propertyType":"msg","rules":[{"t":"cont","v":"NRNZ_NZ_HL_PO","vt":"str"}],"checkall":"false","repair":true,"outputs":1,"x":1020,"y":480,"wires":[["25cc2c197580df6c"]]},{"id":"25cc2c197580df6c","type":"function","z":"b9860b4b9de8c8da","name":"Set filename and localfilename","func":"msg.operation = \"get\";\n\nlet lclFolder = '/media/share/sftpSAPTest/NB-SAP/Inbound/Staging/';\nlet srcfolder = '/outbound/';\nmsg.localFilename = lclFolder + msg.payload.name;\nmsg.filename = srcfolder + msg.payload.name;\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":1250,"y":480,"wires":[["e3b27537257158c1"]]},{"id":"e3b27537257158c1","type":"debug","z":"b9860b4b9de8c8da","name":"debug 11","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":1260,"y":540,"wires":[]}]
I do not have your ftp node installed, so guess as to its output. Maybe you could supply the ftp output in an inject node, if you require further help.
[edit]
You may also want to rate limit the split with a delay node set to rate limit and only output next element once the ftp node has finished each job, you can set a high rate time limit and use msg.flush to send next element. as in this example
Your link put me in mind of where my issue could have been, timing.
I added in a delay timer and managed to get the flow to run.
On more investigation I found node-red-contrib-ftp-download (node) - Node-RED which can take a string input as a file list. This has worked out as well so thanks you for the direction
This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.
