Bulk Rename files via ftp

Hi everyone,

I'm try to rename about 250.000 files using node-red. They are stored in a folder called /artikelbilder and need to be renamed and moved to /renamed based on payload[0] (current filename) and payload[1] (new filename.

I get the correct filenames and am using chunks-to-lines but it won't change and move any of the names.

Any ideas what would be the most efficient way to do this? The images are on a remote server, connected via ftp.
I'd also like to write some file or fill a db with the new filenames, based on what was changed.

Here is my current flow:

[
    {
        "id": "e12aa0fefd18c5f0",
        "type": "tab",
        "label": "Images Name change",
        "disabled": false,
        "info": "",
        "env": []
    },
    {
        "id": "73078a80d05c468d",
        "type": "function",
        "z": "e12aa0fefd18c5f0",
        "name": "",
        "func": "array1 = msg.payload;\nflow.set(\"len\",array1.length);\nfor (var i = 0; i < array1.length; i++){\n    var newMsg = {}\n    newMsg.payload = [array1[i].Artikelnummer, array1[i].EAN];\n    node.send(newMsg);\n}\nreturn null;",
        "outputs": 1,
        "noerr": 0,
        "initialize": "",
        "finalize": "",
        "libs": [],
        "x": 240,
        "y": 340,
        "wires": [
            [
                "376de41b0309bd65"
            ]
        ]
    },
    {
        "id": "376de41b0309bd65",
        "type": "chunks-to-lines",
        "z": "e12aa0fefd18c5f0",
        "name": "",
        "nbLines": "5",
        "linesFormat": "text",
        "decoder": "UTF-8",
        "x": 460,
        "y": 340,
        "wires": [
            [
                "a2540b9deb07295b"
            ]
        ]
    },
    {
        "id": "249d7aacf728c2ee",
        "type": "function",
        "z": "e12aa0fefd18c5f0",
        "name": "Ready for next lines",
        "func": "return [\n    msg.complete ? msg : null,\n    { tick: true },\n];\n",
        "outputs": 2,
        "noerr": 0,
        "initialize": "",
        "finalize": "",
        "x": 1160,
        "y": 280,
        "wires": [
            [],
            [
                "376de41b0309bd65"
            ]
        ]
    },
    {
        "id": "8ab03cb1ef5e85ea",
        "type": "function",
        "z": "e12aa0fefd18c5f0",
        "name": "waiting",
        "func": "let msgCount = context.get(\"msgCount\") || 0;\nlet sendMessage = false;\nmsgCount++; //increment count\nnode.status(\"Count of payload : \" + msgCount);\ncontext.set(\"msgCount\", msgCount); //remember count\nsize = flow.get(\"len\");\nif(msgCount == size){\n    msg.payload = msgCount;\n    return msg;\n}\n\n\n\n",
        "outputs": 1,
        "noerr": 0,
        "initialize": "",
        "finalize": "",
        "x": 1120,
        "y": 220,
        "wires": [
            []
        ]
    },
    {
        "id": "6dfcc25a5a273785",
        "type": "mysql",
        "z": "e12aa0fefd18c5f0",
        "mydb": "d9e35d3d4503fb24",
        "name": "From products",
        "x": 260,
        "y": 280,
        "wires": [
            [
                "73078a80d05c468d"
            ]
        ]
    },
    {
        "id": "561f234787b31687",
        "type": "inject",
        "z": "e12aa0fefd18c5f0",
        "name": "",
        "props": [
            {
                "p": "payload"
            },
            {
                "p": "topic",
                "vt": "str"
            }
        ],
        "repeat": "",
        "crontab": "",
        "once": false,
        "onceDelay": 0.1,
        "topic": "",
        "payload": "",
        "payloadType": "date",
        "x": 180,
        "y": 220,
        "wires": [
            [
                "a735e06674b4e063"
            ]
        ]
    },
    {
        "id": "a2540b9deb07295b",
        "type": "change",
        "z": "e12aa0fefd18c5f0",
        "name": "",
        "rules": [
            {
                "t": "set",
                "p": "operation",
                "pt": "msg",
                "to": "rename",
                "tot": "str"
            },
            {
                "t": "set",
                "p": "oldPath",
                "pt": "msg",
                "to": "\"/artikelbilder/\" & $.payload[0] & \".jpg\"",
                "tot": "jsonata"
            },
            {
                "t": "set",
                "p": "newPath",
                "pt": "msg",
                "to": "\"/renamed/\" & $.payload[1] & \".jpg\"",
                "tot": "jsonata"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 680,
        "y": 340,
        "wires": [
            [
                "10040853bca14472"
            ]
        ]
    },
    {
        "id": "10040853bca14472",
        "type": "advanced-ftp",
        "z": "e12aa0fefd18c5f0",
        "ftp": "980b8fc606b1dfcc",
        "operation": "status",
        "dataType": "binary",
        "filename": "",
        "localFilename": "",
        "workingDir": "",
        "oldPath": "",
        "newPath": "",
        "command": "",
        "recursive": false,
        "useCompression": false,
        "throwError": false,
        "showError": false,
        "name": "",
        "x": 880,
        "y": 220,
        "wires": [
            [
                "249d7aacf728c2ee",
                "8ab03cb1ef5e85ea"
            ]
        ]
    },
    {
        "id": "32e1a0dfe8a9167a",
        "type": "debug",
        "z": "e12aa0fefd18c5f0",
        "name": "",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "false",
        "statusVal": "",
        "statusType": "auto",
        "x": 970,
        "y": 80,
        "wires": []
    },
    {
        "id": "eb1e3399d79f662d",
        "type": "debug",
        "z": "e12aa0fefd18c5f0",
        "name": "",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "true",
        "targetType": "full",
        "statusVal": "",
        "statusType": "auto",
        "x": 450,
        "y": 440,
        "wires": []
    },
    {
        "id": "a735e06674b4e063",
        "type": "function",
        "z": "e12aa0fefd18c5f0",
        "name": "",
        "func": "msg.topic = 'SELECT EAN,LL,LW from product where EAN != \"\" ORDER BY EAN ';\nreturn msg;",
        "outputs": 1,
        "noerr": 0,
        "initialize": "",
        "finalize": "",
        "x": 560,
        "y": 180,
        "wires": [
            [
                "6dfcc25a5a273785"
            ]
        ]
    },
    {
        "id": "91da3c3c0b6b56e8",
        "type": "debug",
        "z": "e12aa0fefd18c5f0",
        "name": "",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "false",
        "statusVal": "",
        "statusType": "auto",
        "x": 550,
        "y": 260,
        "wires": []
    },
    {
        "id": "d9e35d3d4503fb24",
        "type": "MySQLdatabase",
        "name": "dbname",
        "host": "dbhost",
        "port": "3306",
        "db": "dbname2",
        "tz": "",
        "charset": "UTF8"
    },
    {
        "id": "980b8fc606b1dfcc",
        "type": "advanced-ftp-config",
        "host": "servername",
        "port": "",
        "secure": true,
        "secureOptions": "",
        "user": "username",
        "connTimeout": "",
        "pasvTimeout": "",
        "keepalive": "100000",
        "name": "somehost"
    }
]

Thanks a lot!

BTW: if absolutely necessary, I'd also be able to ssh into the remote machine.

Whether absolutely necessary or not, personally I would choose that option over ftp every time. Setup access via keys and run commands on the remote machine using ssh to do what you want. Much simpler and more reliable.

1 Like

Thanks, how would you set this up?

I'm thinking about setting

"mv /artikelbilder/" & $.payload[0] & ".jpg" & " /renamed/" & $.payload[1] & ".jpg"

with a change node and using this as a command for bigssh.

Also tried bigexec before bigssh but this only gives me "Return code 1".

Any idea what would be the fastest method, considering the number of mv commands? (250.000)

Thanks

Is there not some pattern in what needs to be renamed so that you can do them all, or lots of them, in one command?

When I have had to do this I have just used an ssh command to run the command I want, in an exec node.