Workload of 500MB over HTTP Request and MQTT

[thejollypop]

Hello everyone,

I have large files (buffers), 500mb each, I wish to stream/encode them as Base64 array and calculate their length and size, over the HTTP request node. This works well on workloads smaller than 100mb but crashes NR when injecting a 500mb file. I'm trying to workaround this through MQTT and overcome the 256mb limit.

Assistance is greatly appreciated!

My flow:

[{"id":"1c215438d48f35aa","type":"tab","label":"Flow 2","disabled":false,"info":"","env":[]},{"id":"51cdd9eb19b46116","type":"http request","z":"1c215438d48f35aa","name":"","method":"GET","ret":"bin","paytoqs":"ignore","url":"{{{payload}}}","tls":"","persist":false,"proxy":"","authType":"","senderr":false,"x":310,"y":160,"wires":[["43a29608901fbc7a"]]},{"id":"43a29608901fbc7a","type":"split","z":"1c215438d48f35aa","name":"","splt":"100000","spltType":"len","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":450,"y":160,"wires":[["f34213cdd3f624c5"]]},{"id":"66c2fd1271ca0f31","type":"inject","z":"1c215438d48f35aa","name":"500MB file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/4a97c96a26cd01884982487fb396ce7c+LfcgbF6wjMo033pBNGJvoVKEuZ7RKZrbQnUNC.U2niIc9xDjI6NMLGC7Ry4LOFAFIyNAs2iNf2AVg08BY81DKzX4fXWjT8dUjaH5hXSuUZ6CPgp8hQwfsCJkzZ0rhlC2","payloadType":"str","x":140,"y":120,"wires":[["51cdd9eb19b46116"]]},{"id":"52a2658eb066e54c","type":"inject","z":"1c215438d48f35aa","name":"30MB file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/97ebe7c598a8e65d25163f7479a41ce9+p_XgHic1QSeJcp8EKi3XOtbKdXeqNctrK8YLd2wVc413tYgElWNmYq0X6pKDJyKGOQbLcd5lP0LCvUO5HaIg9KZaPcc9Fgubbz6iN_85oSAYkD7tzAlOuj76VIjLNSLy","payloadType":"str","x":140,"y":160,"wires":[["51cdd9eb19b46116"]]},{"id":"d84bffdc47782828","type":"inject","z":"1c215438d48f35aa","name":"92Mb file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/6feaab09a1411cfb1feaebfd8358eb74+kCPor2otXU2yz_BwXI6FfN61w_YxipgYzaRBC13B1rOJuyg.AVkTKpOn.8759NdHZNQv.CxjgMIfZIB9HL4i4iJIG6bkh7Q5uVY7VDDcjiamA0kQ0hFPpd3waK7N5T4B","payloadType":"str","x":160,"y":220,"wires":[["51cdd9eb19b46116"]]},{"id":"f34213cdd3f624c5","type":"base64","z":"1c215438d48f35aa","name":"","action":"","property":"payload","x":580,"y":160,"wires":[["e3020d85c1f0b5b2","0457128bb1cd79da"]]},{"id":"e3020d85c1f0b5b2","type":"join","z":"1c215438d48f35aa","name":"","mode":"custom","build":"array","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"","count":"","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":710,"y":160,"wires":[["7c1e7515d90031e0"]]},{"id":"7c1e7515d90031e0","type":"debug","z":"1c215438d48f35aa","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":850,"y":160,"wires":[]},{"id":"e4b5c24e7885b75d","type":"mqtt out","z":"1c215438d48f35aa","name":"","topic":"workload_test","qos":"","retain":"","respTopic":"","contentType":"","userProps":"","correl":"","expiry":"","broker":"ed237c6b.40d1b","x":760,"y":420,"wires":[]},{"id":"d1c438eb90c46f26","type":"mqtt in","z":"1c215438d48f35aa","name":"","topic":"workload_test","qos":"2","datatype":"auto","broker":"ed237c6b.40d1b","nl":false,"rap":true,"rh":0,"inputs":0,"x":370,"y":560,"wires":[["7e70b9d42a5064c5"]]},{"id":"889226a2366ee7bc","type":"debug","z":"1c215438d48f35aa","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":810,"y":560,"wires":[]},{"id":"a90f8f9fed597b11","type":"inject","z":"1c215438d48f35aa","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"hello world","payloadType":"str","x":560,"y":380,"wires":[["e4b5c24e7885b75d"]]},{"id":"ac316d8458e31d3b","type":"inject","z":"1c215438d48f35aa","name":"80Mb file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/1c62af7137fb8dd77b53b7db30d51b00+KF0QwCcjJFsE7WihvoUkaOEhK_1Vrz8stYGQAilyE9420eEL7FHCUotv287Sibhvedz3R58yZkvCoaFlMCpHa0iejOJMyCHLA7OjeYOApuUKIJ6XZCzN8sVkUnsLoNrE ","payloadType":"str","x":170,"y":280,"wires":[["51cdd9eb19b46116"]]},{"id":"0457128bb1cd79da","type":"debug","z":"1c215438d48f35aa","name":"","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":690,"y":220,"wires":[]},{"id":"4b4a73cdf6b4483b","type":"http request","z":"1c215438d48f35aa","name":"","method":"GET","ret":"bin","paytoqs":"ignore","url":"{{{payload}}}","tls":"","persist":false,"proxy":"","authType":"","senderr":false,"x":310,"y":440,"wires":[["656654c2a23d41ab"]]},{"id":"656654c2a23d41ab","type":"split","z":"1c215438d48f35aa","name":"","splt":"100000","spltType":"len","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":450,"y":440,"wires":[["591493f638e1f465"]]},{"id":"823cb64c77a77371","type":"inject","z":"1c215438d48f35aa","name":"500MB file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/4a97c96a26cd01884982487fb396ce7c+LfcgbF6wjMo033pBNGJvoVKEuZ7RKZrbQnUNC.U2niIc9xDjI6NMLGC7Ry4LOFAFIyNAs2iNf2AVg08BY81DKzX4fXWjT8dUjaH5hXSuUZ6CPgp8hQwfsCJkzZ0rhlC2","payloadType":"str","x":140,"y":380,"wires":[["4b4a73cdf6b4483b"]]},{"id":"e0fe8c702521e965","type":"inject","z":"1c215438d48f35aa","name":"30MB file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/97ebe7c598a8e65d25163f7479a41ce9+p_XgHic1QSeJcp8EKi3XOtbKdXeqNctrK8YLd2wVc413tYgElWNmYq0X6pKDJyKGOQbLcd5lP0LCvUO5HaIg9KZaPcc9Fgubbz6iN_85oSAYkD7tzAlOuj76VIjLNSLy","payloadType":"str","x":140,"y":440,"wires":[["4b4a73cdf6b4483b"]]},{"id":"de9a0cb7fcbcc26e","type":"inject","z":"1c215438d48f35aa","name":"92Mb file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/6feaab09a1411cfb1feaebfd8358eb74+kCPor2otXU2yz_BwXI6FfN61w_YxipgYzaRBC13B1rOJuyg.AVkTKpOn.8759NdHZNQv.CxjgMIfZIB9HL4i4iJIG6bkh7Q5uVY7VDDcjiamA0kQ0hFPpd3waK7N5T4B","payloadType":"str","x":160,"y":500,"wires":[["4b4a73cdf6b4483b"]]},{"id":"b7b19e9d3304dcda","type":"inject","z":"1c215438d48f35aa","name":"80Mb file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/1c62af7137fb8dd77b53b7db30d51b00+KF0QwCcjJFsE7WihvoUkaOEhK_1Vrz8stYGQAilyE9420eEL7FHCUotv287Sibhvedz3R58yZkvCoaFlMCpHa0iejOJMyCHLA7OjeYOApuUKIJ6XZCzN8sVkUnsLoNrE ","payloadType":"str","x":170,"y":560,"wires":[["4b4a73cdf6b4483b"]]},{"id":"7e70b9d42a5064c5","type":"join","z":"1c215438d48f35aa","name":"","mode":"custom","build":"array","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"10","count":"","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":530,"y":560,"wires":[["889226a2366ee7bc"]]},{"id":"591493f638e1f465","type":"base64","z":"1c215438d48f35aa","name":"","action":"","property":"payload","x":580,"y":440,"wires":[["e4b5c24e7885b75d"]]},{"id":"b99ab97193f573fc","type":"comment","z":"1c215438d48f35aa","name":"Without MQTT","info":"","x":320,"y":80,"wires":[]},{"id":"e514964966978b29","type":"comment","z":"1c215438d48f35aa","name":"With MQTT","info":"","x":310,"y":360,"wires":[]},{"id":"ed237c6b.40d1b","type":"mqtt-broker","name":"test.mosquitto.org","broker":"test.mosquitto.org","port":"1883","tls":"ce0f0716.0b63b8","clientid":"","autoConnect":true,"usetls":false,"protocolVersion":"4","keepalive":"60","cleansession":true,"birthTopic":"","birthQos":"0","birthPayload":"","birthMsg":{},"closeTopic":"","closeQos":"0","closePayload":"","closeMsg":{},"willTopic":"","willQos":"0","willPayload":"","willMsg":{},"sessionExpiry":""},{"id":"ce0f0716.0b63b8","type":"tls-config","name":"","cert":"","key":"","ca":"","certname":"","keyname":"","caname":"","servername":"","verifyservercert":true}]

[shrickus]

I did not import this flow and try it -- but, it looks like you are trying to split a 500Mb buffer into 5,000 msg objects with 100,000 byte each? Then each msg object is made at least 35% larger by being encoded (and perhaps double that if there is any cloning of the data)... it's not unexpected to run out of resources. When node-red crashes, I'm assuming the error is an out of memory condition? How big is the hardware you are running, and which OS?

Interesting problem... Depending on your OS, there may be a command-line program that you can invoke that would be way more efficient -- save the file to disk, use an exec node to convert it, and return the encoded file contents.

Another thought would be to use the split node in "streaming" mode, with some rate limiting or feedback loop to allow each chunk to finish processing before starting the next. Of course, that has the added complexity of knowing when the stream is finished, and verifying that all the data has been processed. I have been successfully using the q-gate and semaphore nodes lately, while trying to maximize throughput without overloading the system.

I'm sure there are dozens of ways to handle such large files (ftp with a file watcher maybe?), but those are my initial thoughts and questions. Please document your successes here so the rest of us can learn as well... good luck!

[shrickus]

In fact, I think I missed the easiest solution -- if you can store the uploaded file to a temp location, use the read file node, set encoding as 'base64', read in the temp file and return it... done?

[thejollypop]

Thanks @shrickus, As you suggested, I used the write file and read file nodes with Base64 encoding as input and output, and appended them together as a string or array. This seems faster and more scalable, with option 2 being a preferable in terms of functionality, but still when injecting 542mb payload, NR crashes.
I'm running NR on Heroku and also on Alpine Linux and Debian dockers. All have the same behavior when it comes to large payloads. As you mentioned I tried also smaller chunks with the split node, with fixed length of 1000, but nothing but crashes. Thoughts? Thanks

New flow, based on file nodes:

[{"id":"170a60238eef6198","type":"group","z":"5dad8ffdcd25615d","style":{"stroke":"#999999","stroke-opacity":"1","fill":"none","fill-opacity":"1","label":true,"label-position":"nw","color":"#a4a4a4"},"nodes":["f7f556c97bd5e198","38a0023e616b4055","a1127578e35ccc8e","e73d59da8de64749","012bfbe0039a5f1f","978d0c775846c895","8c5cab429c7c2538","2d346f2bcd9145ff","ae93749b73cdfbf6","65354b7351eaca65","96ad589bbd7b5b2b","908c97ab1f1069a1","9be1bcd519082b2d","b0687acec400c427","05ecf598cfd0e6ee","bdb76579e728e8c0","f208a0b1437d87ec"],"x":34,"y":479,"w":1152,"h":382},{"id":"f7f556c97bd5e198","type":"inject","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"true","payloadType":"bool","x":130,"y":580,"wires":[["38a0023e616b4055"]]},{"id":"38a0023e616b4055","type":"file","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","filename":"output.txt","appendNewline":true,"createDir":false,"overwriteFile":"delete","encoding":"none","x":370,"y":580,"wires":[["a1127578e35ccc8e"]]},{"id":"a1127578e35ccc8e","type":"debug","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":550,"y":580,"wires":[]},{"id":"e73d59da8de64749","type":"inject","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"80Mb file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/1c62af7137fb8dd77b53b7db30d51b00+KF0QwCcjJFsE7WihvoUkaOEhK_1Vrz8stYGQAilyE9420eEL7FHCUotv287Sibhvedz3R58yZkvCoaFlMCpHa0iejOJMyCHLA7OjeYOApuUKIJ6XZCzN8sVkUnsLoNrE ","payloadType":"str","x":140,"y":680,"wires":[["2d346f2bcd9145ff"]]},{"id":"012bfbe0039a5f1f","type":"file","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","filename":"output.txt","appendNewline":true,"createDir":false,"overwriteFile":"false","encoding":"base64","x":600,"y":700,"wires":[["978d0c775846c895"]]},{"id":"978d0c775846c895","type":"file in","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","filename":"output.txt","format":"lines","chunk":false,"sendError":false,"encoding":"base64","allProps":false,"x":740,"y":700,"wires":[["b0687acec400c427"]]},{"id":"8c5cab429c7c2538","type":"debug","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1030,"y":600,"wires":[]},{"id":"2d346f2bcd9145ff","type":"http request","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","method":"GET","ret":"bin","paytoqs":"ignore","url":"{{{payload}}}","tls":"","persist":false,"proxy":"","authType":"","senderr":false,"x":310,"y":700,"wires":[["96ad589bbd7b5b2b"]]},{"id":"ae93749b73cdfbf6","type":"inject","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"542MB file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/e58715e1bf5050d6d0381c341acc555b+OK1Mp3jL9nRdNoAcpXVpQUpWjP85u.fd_s5vF9DnMznbo9.VqNtDMPaQDcgxCpY5aAtYhh72naJ9EdgXhE1hAjsjKqzrK3KXMgIKeFxgNUr7ndILKE1o4_I5x2ucqMl9","payloadType":"str","x":160,"y":820,"wires":[["2d346f2bcd9145ff"]]},{"id":"65354b7351eaca65","type":"inject","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"27mb file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/845986a2ad5860dcb76f72e3a04f5a25+usTgy8kqYC3bEn2tMVN65uHegQ4X6vFliYUJlJ9P5nS5ljLbcwWx7yAkN9BrCB02xW3fTSWLAzhhyJDkyaJCXLvRu0Gbk3bQcYiSQ2uW8I44oGRWoAvRPJUf0XqLJboW","payloadType":"str","x":140,"y":740,"wires":[["2d346f2bcd9145ff"]]},{"id":"96ad589bbd7b5b2b","type":"base64","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","action":"","property":"payload","x":460,"y":700,"wires":[["012bfbe0039a5f1f"]]},{"id":"908c97ab1f1069a1","type":"join","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"Array","mode":"reduce","build":"string","property":"payload","propertyType":"msg","key":"topic","joiner":",","joinerType":"str","accumulate":false,"timeout":"","count":"2","reduceRight":false,"reduceExp":"$append($A,[payload])","reduceInit":"[]","reduceInitType":"json","reduceFixup":"","x":870,"y":640,"wires":[["8c5cab429c7c2538","9be1bcd519082b2d"]]},{"id":"9be1bcd519082b2d","type":"debug","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload.length","targetType":"msg","statusVal":"","statusType":"auto","x":1050,"y":680,"wires":[]},{"id":"b0687acec400c427","type":"join","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"String","mode":"custom","build":"string","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"","count":"","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":870,"y":760,"wires":[["05ecf598cfd0e6ee","bdb76579e728e8c0"]]},{"id":"05ecf598cfd0e6ee","type":"debug","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1030,"y":720,"wires":[]},{"id":"bdb76579e728e8c0","type":"debug","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload.length","targetType":"msg","statusVal":"","statusType":"auto","x":1050,"y":800,"wires":[]},{"id":"f208a0b1437d87ec","type":"comment","z":"5dad8ffdcd25615d","g":"170a60238eef6198","name":"Option 2 - Base64 via Base64 Node","info":"","x":200,"y":520,"wires":[]},{"id":"282cf794d4bcd2c9","type":"group","z":"5dad8ffdcd25615d","style":{"stroke":"#999999","stroke-opacity":"1","fill":"none","fill-opacity":"1","label":true,"label-position":"nw","color":"#a4a4a4"},"nodes":["adde4d2a7c4f7862","ee95f60b9dd68327","ffbd66aa9e1daccb","bff9d327f54945c6","b7defe17ed684751","98f4f562fe4c9688","80c8853be840bd64","6fac4f4d46409914","bf1fa891e4c36d67","80cebcde1109c947","52f240dbf0b09e46"],"x":34,"y":79,"w":872,"h":382},{"id":"adde4d2a7c4f7862","type":"inject","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"80Mb file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/1c62af7137fb8dd77b53b7db30d51b00+KF0QwCcjJFsE7WihvoUkaOEhK_1Vrz8stYGQAilyE9420eEL7FHCUotv287Sibhvedz3R58yZkvCoaFlMCpHa0iejOJMyCHLA7OjeYOApuUKIJ6XZCzN8sVkUnsLoNrE ","payloadType":"str","x":160,"y":200,"wires":[["b7defe17ed684751"]]},{"id":"ee95f60b9dd68327","type":"file","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"","filename":"output.txt","appendNewline":false,"createDir":false,"overwriteFile":"false","encoding":"base64","x":480,"y":240,"wires":[["ffbd66aa9e1daccb"]]},{"id":"ffbd66aa9e1daccb","type":"file in","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"","filename":"output.txt","format":"utf8","chunk":false,"sendError":false,"encoding":"base64","allProps":false,"x":640,"y":240,"wires":[["bff9d327f54945c6"]]},{"id":"bff9d327f54945c6","type":"debug","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":790,"y":240,"wires":[]},{"id":"b7defe17ed684751","type":"http request","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"","method":"GET","ret":"bin","paytoqs":"ignore","url":"{{{payload}}}","tls":"","persist":false,"proxy":"","authType":"","senderr":false,"x":330,"y":240,"wires":[["ee95f60b9dd68327"]]},{"id":"98f4f562fe4c9688","type":"inject","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"2MB file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/845986a2ad5860dcb76f72e3a04f5a25+usTgy8kqYC3bEn2tMVN65uHegQ4X6vFliYUJlJ9P5nS5ljLbcwWx7yAkN9BrCB02xW3fTSWLAzhhyJDkyaJCXLvRu0Gbk3bQcYiSQ2uW8I44oGRWoAvRPJUf0XqLJboW","payloadType":"str","x":140,"y":260,"wires":[["b7defe17ed684751"]]},{"id":"80c8853be840bd64","type":"comment","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"Option 1 - Base64 via File Node","info":"","x":290,"y":120,"wires":[]},{"id":"6fac4f4d46409914","type":"inject","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"true","payloadType":"bool","x":370,"y":420,"wires":[["bf1fa891e4c36d67"]]},{"id":"bf1fa891e4c36d67","type":"file","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"","filename":"output.txt","appendNewline":true,"createDir":false,"overwriteFile":"delete","encoding":"none","x":530,"y":420,"wires":[["80cebcde1109c947"]]},{"id":"80cebcde1109c947","type":"debug","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":710,"y":420,"wires":[]},{"id":"52f240dbf0b09e46","type":"inject","z":"5dad8ffdcd25615d","g":"282cf794d4bcd2c9","name":"542MB file","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://tusd.tusdemo.net/files/e58715e1bf5050d6d0381c341acc555b+OK1Mp3jL9nRdNoAcpXVpQUpWjP85u.fd_s5vF9DnMznbo9.VqNtDMPaQDcgxCpY5aAtYhh72naJ9EdgXhE1hAjsjKqzrK3KXMgIKeFxgNUr7ndILKE1o4_I5x2ucqMl9","payloadType":"str","x":140,"y":320,"wires":[["b7defe17ed684751"]]}]

[shrickus]

I don't think I suggested to use any split nodes... just upload the file using http in, save it to disk with file write, then use the file read node with "base64" encoding. What do you plan to do with this uploaded file? I'm only asking because if you need to send the file data somewhere outside of node-red, you might want to avoid dragging it all back into memory just to push it to some other file system.

You have not said what error is thrown -- but I'm assuming it is an out of memory problem. The more processing you do inside the flow, the more memory you will need. And containers are often configured with very low memory footprints...

[thejollypop]

Ok, the error type was 'out of memory', but now I'm able to save the 542mb payload into a file write node without any crash(!). I've disconnected the debug output node and applied reading a message per line. My goal is to use JSONata string manipulations, like $substring , $replace, $trim and more. Still I have to send the message after a timeout following the first message to control and summarize the output somehow, but I believe the 500mb payload barrier is now reachable. Thanks again. Flow:

[{"id":"a07509107c7eda8b","type":"tab","label":"Flow 1","disabled":false,"info":"","env":[]},{"id":"22bc378dc3dfbf3d","type":"inject","z":"a07509107c7eda8b","name":"mark","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"https://i.insider.com/6154dfbbb414c100186301fb?width=1136&format=jpeg","payloadType":"str","x":290,"y":360,"wires":[["6dab2d58dd9f692a"]]},{"id":"6dab2d58dd9f692a","type":"http request","z":"a07509107c7eda8b","name":"","method":"GET","ret":"bin","paytoqs":"ignore","url":"{{{payload}}}","tls":"","persist":false,"proxy":"","authType":"","senderr":false,"x":390,"y":420,"wires":[["580ef4d77d9f9b7f"]]},{"id":"b33508421a3b700c","type":"file","z":"a07509107c7eda8b","name":"","filename":"output.tmp","appendNewline":true,"createDir":false,"overwriteFile":"true","encoding":"none","x":730,"y":420,"wires":[["b03c8df45e48d118"]]},{"id":"b03c8df45e48d118","type":"file in","z":"a07509107c7eda8b","name":"","filename":"output.tmp","format":"lines","chunk":false,"sendError":false,"encoding":"none","allProps":false,"x":890,"y":420,"wires":[["d370535fada6f05c","7a5efd8287c88245","325ce98bd30382fc"]]},{"id":"d370535fada6f05c","type":"debug","z":"a07509107c7eda8b","name":"","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1150,"y":420,"wires":[]},{"id":"a4d49e25812d9f48","type":"inject","z":"a07509107c7eda8b","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"true","payloadType":"bool","x":430,"y":560,"wires":[["0a599d4b552ba438"]]},{"id":"0a599d4b552ba438","type":"file","z":"a07509107c7eda8b","name":"DELETE","filename":"output.tmp","appendNewline":false,"createDir":false,"overwriteFile":"delete","encoding":"base64","x":560,"y":560,"wires":[["fdf6b1843eb7f1c9"]]},{"id":"fdf6b1843eb7f1c9","type":"debug","z":"a07509107c7eda8b","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":730,"y":560,"wires":[]},{"id":"e35410b452009f16","type":"image viewer","z":"a07509107c7eda8b","name":"","width":160,"data":"payload","dataType":"msg","active":true,"x":1210,"y":320,"wires":[[]]},{"id":"7a5efd8287c88245","type":"jimp-image","z":"a07509107c7eda8b","name":"","data":"payload","dataType":"msg","ret":"buf","parameter1":"","parameter1Type":"msg","parameter2":"","parameter2Type":"msg","parameter3":"","parameter3Type":"msg","parameter4":"","parameter4Type":"msg","parameter5":"","parameter5Type":"msg","parameter6":"","parameter6Type":"msg","parameter7":"","parameter7Type":"msg","parameter8":"","parameter8Type":"msg","sendProperty":"payload","sendPropertyType":"msg","parameterCount":0,"jimpFunction":"none","selectedJimpFunction":{"name":"none","fn":"none","description":"Just loads the image.","parameters":[]},"x":1050,"y":320,"wires":[["e35410b452009f16"]]},{"id":"580ef4d77d9f9b7f","type":"base64","z":"a07509107c7eda8b","name":"","action":"str","property":"payload","x":560,"y":420,"wires":[["b33508421a3b700c"]]},{"id":"aee97dfcc40f4e9f","type":"debug","z":"a07509107c7eda8b","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1230,"y":500,"wires":[]},{"id":"325ce98bd30382fc","type":"join","z":"a07509107c7eda8b","name":"","mode":"custom","build":"string","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"10","count":"","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":990,"y":500,"wires":[["aee97dfcc40f4e9f","064ea1407e012c53"]]},{"id":"064ea1407e012c53","type":"change","z":"a07509107c7eda8b","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"$replace(\t   payload,\t   \"CEQEDEQH/xAAbAAADAQEBAQEAAAAAAAAAAAAAAQIDBAUGB//E\",\t   \"!\"\t)","tot":"jsonata"}],"action":"","property":"","from":"","to":"","reg":false,"x":1100,"y":580,"wires":[["37d3a1996736c0e7"]]},{"id":"37d3a1996736c0e7","type":"debug","z":"a07509107c7eda8b","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload.length","targetType":"msg","statusVal":"","statusType":"auto","x":1330,"y":580,"wires":[]}]