Running a join node after filtering my messages

General
1

Hi everyone,

I just encounter a problem that has already happened to me.
My objective is to convert french payroll files to a format that the PERP of my company accept.

The problem is : after filtering and convert my messages, the join node transmits nothing to its output.

I think its linked to the fact that some messages are missing (cause of the filtering).

There is my flow where I've added a way for you to test it with some data :

[{"id":"487be162.3f1b8","type":"tab","label":"Payroll files from PayFit","disabled":false,"info":""},{"id":"999aed54.e990e","type":"change","z":"487be162.3f1b8","name":"Converter","rules":[{"t":"set","p":"payload.EC_NUMERO","pt":"msg","to":"file","tot":"msg"},{"t":"set","p":"payload","pt":"msg","to":"$$.payload.{\t    \"EC_NUMERO\": $.EC_NUMERO,\t    \"EC_DATE\": $.EcritureDate,\t    \"EC_CODE_JOURNAL\": \"PA\",\t    \"EC_EXERCICE\": 24,\t    \"LI_COMPTE\": $.CompteNum,\t    \"LI_LIBELLE\": $.CompteLib,\t    \"LI_DEBIT\": ($.Debit or $.Debit = 0 or $.Debit = \"0\") ? $number($.Debit) : \"\",\t    \"LI_CREDIT\": ($.Credit or $.Credit = 0 or $.Credit = \"0\") ? $number($.Credit) : \"\",\t    \"LI_DEVISE\": \"EUR\",\t    \"LI_PIECE\": $.EC_NUMERO,\t    \"LI_AFFAIRE\": $.AxeReference\t}","tot":"jsonata"}],"action":"","property":"","from":"","to":"","reg":false,"x":920,"y":280,"wires":[["fd470a8d.6ce038","9ea450d1.1807f"]]},{"id":"ca62b93b.be38b8","type":"function","z":"487be162.3f1b8","name":"Filter empty or null lines","func":"// Extraire les valeurs de débit et de crédit du message\nlet debit = msg.payload.Debit;\nlet credit = msg.payload.Credit;\n\nif ((debit === 0 || debit === \"0\" || debit === \"\") && (credit === 0 || credit === \"0\" || credit === \"\")) {\n    return null; // Ne pas transmettre le message si les conditions sont remplies\n} else if ((typeof debit === 'undefined' || debit === null || debit === \"\") && (credit === 0 || credit === \"0\" || credit === \"\")) {\n    return null; // Ne pas transmettre le message si debit est undefined, null ou vide et credit est 0, \"0\" ou vide\n} else if ((debit === 0 || debit === \"0\" || debit === \"\") && (typeof credit === 'undefined' || credit === null || credit === \"\")) {\n    return null; // Ne pas transmettre le message si credit est undefined, null ou vide et debit est 0, \"0\" ou vide\n} else if ((typeof debit === 'undefined' || debit === null || debit === \"\") && (typeof credit === 'undefined' || credit === null || credit === \"\")) {\n    return null; // Ne pas transmettre le message si debit et credit sont undefined, null ou vide\n} else {\n    return msg;\n}","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":710,"y":280,"wires":[["999aed54.e990e","bf1e6320.51cb2"]]},{"id":"83d786e7.44c868","type":"watch","z":"487be162.3f1b8","name":"Controller","files":"braincubeAppsData/files/Payroll files from PayFit/Converter deposit","event":"add","usePolling":false,"hidden":false,"x":180,"y":220,"wires":[["c0a24273.93e8c","2d6ac03d.38e3f"]]},{"id":"bca4efe9.57c36","type":"file in","z":"487be162.3f1b8","name":"Payroll","filename":"","format":"utf8","chunk":false,"sendError":false,"encoding":"none","x":170,"y":340,"wires":[["e3f1352f.019728"]]},{"id":"c0a24273.93e8c","type":"debug","z":"487be162.3f1b8","name":"","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":330,"y":220,"wires":[]},{"id":"2d6ac03d.38e3f","type":"change","z":"487be162.3f1b8","name":"Define filename","rules":[{"t":"set","p":"filename","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":200,"y":280,"wires":[["bca4efe9.57c36"]]},{"id":"957b47ce.dc6d28","type":"csv","z":"487be162.3f1b8","name":"","sep":";","hdrin":true,"hdrout":"none","multi":"one","ret":"\\n","temp":"","skip":"0","strings":true,"include_empty_strings":"","include_null_values":"","x":510,"y":280,"wires":[["cb2b1918.600328","ca62b93b.be38b8"]]},{"id":"cb2b1918.600328","type":"debug","z":"487be162.3f1b8","name":"Base data","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":660,"y":340,"wires":[]},{"id":"e3f1352f.019728","type":"function","z":"487be162.3f1b8","name":"Redefine msg.file","func":"let filename = msg.file;\nlet transformedFilename = filename.replace('-', '/').replace('.csv', '');\n\nmsg.file = transformedFilename;\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":370,"y":340,"wires":[["957b47ce.dc6d28"]]},{"id":"fd470a8d.6ce038","type":"debug","z":"487be162.3f1b8","name":"Converted data","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1100,"y":280,"wires":[]},{"id":"bf1e6320.51cb2","type":"debug","z":"487be162.3f1b8","name":"Filtered data","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":930,"y":220,"wires":[]},{"id":"54c64587.dc851c","type":"debug","z":"487be162.3f1b8","name":"","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":1390,"y":400,"wires":[]},{"id":"9ea450d1.1807f","type":"join","z":"487be162.3f1b8","name":"","mode":"custom","build":"array","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"","count":"","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":1070,"y":340,"wires":[["ec5c513f.21b3d","d40fda46.f47488"]]},{"id":"ec5c513f.21b3d","type":"function","z":"487be162.3f1b8","name":"Set filename","func":"let filename = msg.file;\nlet transformedFilename = filename.replace('/', '-');\n\nmsg.file = transformedFilename;\nmsg.filename = '/var/lib/braincube-edge/.node-red/braincubeAppsData/files/Payroll files from PayFit/'+ msg.file + '.csv';\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":1090,"y":400,"wires":[["5df58932.d69de8"]]},{"id":"5df58932.d69de8","type":"csv","z":"487be162.3f1b8","name":"","sep":",","hdrin":"","hdrout":"all","multi":"mult","ret":"\\n","temp":"EC_NUMERO,EC_DATE,EC_CODE_JOURNAL,EC_EXERCICE,LI_COMPTE,LI_LIBELLE,LI_DEBIT,LI_CREDIT,LI_DEVISE,LI_PIECE,LI_AFFAIRE","skip":"0","strings":true,"include_empty_strings":true,"include_null_values":true,"x":1250,"y":400,"wires":[["54c64587.dc851c","f5cfa56d.c479e8"]]},{"id":"f5cfa56d.c479e8","type":"file","z":"487be162.3f1b8","name":"Converted payrolls","filename":"","appendNewline":true,"createDir":false,"overwriteFile":"true","encoding":"none","x":1430,"y":340,"wires":[[]]},{"id":"28ecef80.f7d3","type":"inject","z":"487be162.3f1b8","name":"","props":[{"p":"payload"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437300,\"CompteLib\":\"Prevoyance - charges patronales\",\"Credit\":0,\"AxeLib\":\"Operations infrastructure engineering\",\"AxeReference\":\"B000069\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":641400,\"CompteLib\":\"Avantages en nature\",\"Debit\":103.2,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":421000,\"CompteLib\":\"Personnel - remunerations dues\",\"Credit\":0,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437200,\"CompteLib\":\"Caisse de retraite AGIRC-ARRCO - charges salariales\",\"Credit\":234.94,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437300,\"CompteLib\":\"Prevoyance - charges salariales\",\"Credit\":0,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437300,\"CompteLib\":\"Mutuelle - charges salariales\",\"Credit\":54.98,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":791000,\"CompteLib\":\"Avantages en nature\",\"Credit\":103.2,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":442100,\"CompteLib\":\"Prelevement a la source\",\"Credit\":208.83,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":645210,\"CompteLib\":\"Cotisations prevoyance\",\"Debit\":88.17,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":631200,\"CompteLib\":\"Taxe d'apprentissage\",\"Debit\":39.71,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"}]","payloadType":"json","x":510,"y":160,"wires":[["1d65d817.0fe2f8"]]},{"id":"1d65d817.0fe2f8","type":"split","z":"487be162.3f1b8","name":"","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":510,"y":220,"wires":[["ca62b93b.be38b8","6781a04.dfc0e6"]]},{"id":"6781a04.dfc0e6","type":"debug","z":"487be162.3f1b8","name":"Base data","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":660,"y":220,"wires":[]},{"id":"d40fda46.f47488","type":"debug","z":"487be162.3f1b8","name":"Converted data","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1175,"y":340,"wires":[],"l":false},{"id":"4c9098d8.5722c8","type":"comment","z":"487be162.3f1b8","name":"You can use the manuel inject","info":"","x":580,"y":120,"wires":[]}]

If someone have a solution that will be a pleasure.
Thanks for reading.

2

Try this, edit the join node and in the After a timeout following the first message option enter 3

This means that if a new message doesn't arrive after 3 seconds, it will join the data it has and pass it onward.

You also have a problem in the 'Set filename' function node. Your first line is:
let filename = msg.file;
but the incoming message does not have msg.file defined.

3

That work !

There is the value of msg.file the moment it detects my file :

Capture d’écran 2024-06-13 à 16.43.20
Capture d’écran 2024-06-13 à 16.43.201973×433 88.6 KB

The problem only shows itself when we use the manual inject but there is an update of my flow where I fix it :

[{"id":"487be162.3f1b8","type":"tab","label":"Payroll files from PayFit","disabled":false,"info":""},{"id":"999aed54.e990e","type":"change","z":"487be162.3f1b8","name":"Converter","rules":[{"t":"set","p":"payload.EC_NUMERO","pt":"msg","to":"file","tot":"msg"},{"t":"set","p":"payload","pt":"msg","to":"$$.payload.{\t    \"EC_NUMERO\": $.EC_NUMERO,\t    \"EC_DATE\": $.EcritureDate,\t    \"EC_CODE_JOURNAL\": \"PA\",\t    \"EC_EXERCICE\": 24,\t    \"LI_COMPTE\": $.CompteNum,\t    \"LI_LIBELLE\": $.CompteLib,\t    \"LI_DEBIT\": ($.Debit or $.Debit = 0 or $.Debit = \"0\") ? $number($.Debit) : \"\",\t    \"LI_CREDIT\": ($.Credit or $.Credit = 0 or $.Credit = \"0\") ? $number($.Credit) : \"\",\t    \"LI_DEVISE\": \"EUR\",\t    \"LI_PIECE\": $.EC_NUMERO,\t    \"LI_AFFAIRE\": $.AxeReference\t}","tot":"jsonata"}],"action":"","property":"","from":"","to":"","reg":false,"x":920,"y":280,"wires":[["fd470a8d.6ce038","9ea450d1.1807f"]]},{"id":"ca62b93b.be38b8","type":"function","z":"487be162.3f1b8","name":"Filter empty or null lines","func":"// Extraire les valeurs de débit et de crédit du message\nlet debit = msg.payload.Debit;\nlet credit = msg.payload.Credit;\n\nif ((debit === 0 || debit === \"0\" || debit === \"\") && (credit === 0 || credit === \"0\" || credit === \"\")) {\n    return null; // Ne pas transmettre le message si les conditions sont remplies\n} else if ((typeof debit === 'undefined' || debit === null || debit === \"\") && (credit === 0 || credit === \"0\" || credit === \"\")) {\n    return null; // Ne pas transmettre le message si debit est undefined, null ou vide et credit est 0, \"0\" ou vide\n} else if ((debit === 0 || debit === \"0\" || debit === \"\") && (typeof credit === 'undefined' || credit === null || credit === \"\")) {\n    return null; // Ne pas transmettre le message si credit est undefined, null ou vide et debit est 0, \"0\" ou vide\n} else if ((typeof debit === 'undefined' || debit === null || debit === \"\") && (typeof credit === 'undefined' || credit === null || credit === \"\")) {\n    return null; // Ne pas transmettre le message si debit et credit sont undefined, null ou vide\n} else {\n    return msg;\n}","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":710,"y":280,"wires":[["999aed54.e990e","bf1e6320.51cb2"]]},{"id":"83d786e7.44c868","type":"watch","z":"487be162.3f1b8","name":"Controller","files":"braincubeAppsData/files/Payroll files from PayFit/Converter deposit","event":"add","usePolling":false,"hidden":false,"x":180,"y":220,"wires":[["c0a24273.93e8c","2d6ac03d.38e3f"]]},{"id":"bca4efe9.57c36","type":"file in","z":"487be162.3f1b8","name":"Payroll","filename":"","format":"utf8","chunk":false,"sendError":false,"encoding":"none","x":170,"y":340,"wires":[["e3f1352f.019728"]]},{"id":"c0a24273.93e8c","type":"debug","z":"487be162.3f1b8","name":"","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":330,"y":220,"wires":[]},{"id":"2d6ac03d.38e3f","type":"change","z":"487be162.3f1b8","name":"Define filename","rules":[{"t":"set","p":"filename","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":200,"y":280,"wires":[["bca4efe9.57c36"]]},{"id":"957b47ce.dc6d28","type":"csv","z":"487be162.3f1b8","name":"","sep":";","hdrin":true,"hdrout":"none","multi":"one","ret":"\\n","temp":"","skip":"0","strings":true,"include_empty_strings":"","include_null_values":"","x":510,"y":280,"wires":[["cb2b1918.600328","ca62b93b.be38b8"]]},{"id":"cb2b1918.600328","type":"debug","z":"487be162.3f1b8","name":"Base data","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":660,"y":340,"wires":[]},{"id":"e3f1352f.019728","type":"function","z":"487be162.3f1b8","name":"Redefine msg.file","func":"let filename = msg.file;\nlet transformedFilename = filename.replace('-', '/').replace('.csv', '');\n\nmsg.file = transformedFilename;\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":370,"y":340,"wires":[["957b47ce.dc6d28"]]},{"id":"fd470a8d.6ce038","type":"debug","z":"487be162.3f1b8","name":"Converted data","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1100,"y":280,"wires":[]},{"id":"bf1e6320.51cb2","type":"debug","z":"487be162.3f1b8","name":"Filtered data","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":930,"y":220,"wires":[]},{"id":"54c64587.dc851c","type":"debug","z":"487be162.3f1b8","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":1390,"y":400,"wires":[]},{"id":"9ea450d1.1807f","type":"join","z":"487be162.3f1b8","name":"","mode":"custom","build":"array","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"1","count":"","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":1070,"y":340,"wires":[["ec5c513f.21b3d","d40fda46.f47488"]]},{"id":"ec5c513f.21b3d","type":"function","z":"487be162.3f1b8","name":"Set filename","func":"let filename = msg.file;\nlet transformedFilename = filename.replace('/', '-');\n\nmsg.file = transformedFilename;\nmsg.filename = '/var/lib/braincube-edge/.node-red/braincubeAppsData/files/Payroll files from PayFit/'+ msg.file + '.csv';\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":1090,"y":400,"wires":[["5df58932.d69de8"]]},{"id":"5df58932.d69de8","type":"csv","z":"487be162.3f1b8","name":"","sep":",","hdrin":"","hdrout":"all","multi":"mult","ret":"\\n","temp":"EC_NUMERO,EC_DATE,EC_CODE_JOURNAL,EC_EXERCICE,LI_COMPTE,LI_LIBELLE,LI_DEBIT,LI_CREDIT,LI_DEVISE,LI_PIECE,LI_AFFAIRE","skip":"0","strings":true,"include_empty_strings":true,"include_null_values":true,"x":1250,"y":400,"wires":[["54c64587.dc851c"]]},{"id":"f5cfa56d.c479e8","type":"file","z":"487be162.3f1b8","name":"Converted payrolls","filename":"","appendNewline":true,"createDir":false,"overwriteFile":"true","encoding":"none","x":1430,"y":340,"wires":[[]]},{"id":"28ecef80.f7d3","type":"inject","z":"487be162.3f1b8","name":"","props":[{"p":"payload"},{"p":"file","v":"AMARANTE 05/24","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437300,\"CompteLib\":\"Prevoyance - charges patronales\",\"Credit\":0,\"AxeLib\":\"Operations infrastructure engineering\",\"AxeReference\":\"B000069\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":641400,\"CompteLib\":\"Avantages en nature\",\"Debit\":103.2,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":421000,\"CompteLib\":\"Personnel - remunerations dues\",\"Credit\":0,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437200,\"CompteLib\":\"Caisse de retraite AGIRC-ARRCO - charges salariales\",\"Credit\":234.94,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437300,\"CompteLib\":\"Prevoyance - charges salariales\",\"Credit\":0,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437300,\"CompteLib\":\"Mutuelle - charges salariales\",\"Credit\":54.98,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":791000,\"CompteLib\":\"Avantages en nature\",\"Credit\":103.2,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":442100,\"CompteLib\":\"Prelevement a la source\",\"Credit\":208.83,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":645210,\"CompteLib\":\"Cotisations prevoyance\",\"Debit\":88.17,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":631200,\"CompteLib\":\"Taxe d'apprentissage\",\"Debit\":39.71,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"}]","payloadType":"json","x":510,"y":160,"wires":[["1d65d817.0fe2f8"]]},{"id":"1d65d817.0fe2f8","type":"split","z":"487be162.3f1b8","name":"","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":510,"y":220,"wires":[["ca62b93b.be38b8","6781a04.dfc0e6"]]},{"id":"6781a04.dfc0e6","type":"debug","z":"487be162.3f1b8","name":"Base data","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":660,"y":220,"wires":[]},{"id":"d40fda46.f47488","type":"debug","z":"487be162.3f1b8","name":"Converted data","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1175,"y":340,"wires":[],"l":false},{"id":"4c9098d8.5722c8","type":"comment","z":"487be162.3f1b8","name":"You can use the manuel inject","info":"","x":580,"y":120,"wires":[]}]

Now I've another problem... I remark that the join node maybe don't see my object :
Capture d’écran 2024-06-13 à 16.58.11
Do you know what it's due to?

4

With your latest flow, when I run it using the manual inject, I don't see an issue.

However I see that when you read in the file you are passing it thru a csv node. What is coming out of that node (change the debug node to display the complete msg object and copy/paste the results to a reply)

5

I think complete attribute option shall help - send last message having complete attribute.
Take a look this example for inspiration on this

[{"id":"999aed54.e990e","type":"change","z":"487be162.3f1b8","name":"Converter","rules":[{"t":"set","p":"payload.EC_NUMERO","pt":"msg","to":"file","tot":"msg"},{"t":"set","p":"payload","pt":"msg","to":"$$.payload.{\t    \"EC_NUMERO\": $.EC_NUMERO,\t    \"EC_DATE\": $.EcritureDate,\t    \"EC_CODE_JOURNAL\": \"PA\",\t    \"EC_EXERCICE\": 24,\t    \"LI_COMPTE\": $.CompteNum,\t    \"LI_LIBELLE\": $.CompteLib,\t    \"LI_DEBIT\": ($.Debit or $.Debit = 0 or $.Debit = \"0\") ? $number($.Debit) : \"\",\t    \"LI_CREDIT\": ($.Credit or $.Credit = 0 or $.Credit = \"0\") ? $number($.Credit) : \"\",\t    \"LI_DEVISE\": \"EUR\",\t    \"LI_PIECE\": $.EC_NUMERO,\t    \"LI_AFFAIRE\": $.AxeReference\t}","tot":"jsonata"}],"action":"","property":"","from":"","to":"","reg":false,"x":920,"y":280,"wires":[["fd470a8d.6ce038","9ea450d1.1807f"]]},{"id":"ca62b93b.be38b8","type":"function","z":"487be162.3f1b8","name":"Filter empty or null lines","func":"// Extraire les valeurs de débit et de crédit du message\nlet debit = msg.payload.Debit;\nlet credit = msg.payload.Credit;\nif (msg.parts.index === 0) {\n    context.set('last_message', undefined)\n}\nfunction filter() {\nif ((debit === 0 || debit === \"0\" || debit === \"\") && (credit === 0 || credit === \"0\" || credit === \"\")) {\n    return null; // Ne pas transmettre le message si les conditions sont remplies\n} else if ((typeof debit === 'undefined' || debit === null || debit === \"\") && (credit === 0 || credit === \"0\" || credit === \"\")) {\n    return null; // Ne pas transmettre le message si debit est undefined, null ou vide et credit est 0, \"0\" ou vide\n} else if ((debit === 0 || debit === \"0\" || debit === \"\") && (typeof credit === 'undefined' || credit === null || credit === \"\")) {\n    return null; // Ne pas transmettre le message si credit est undefined, null ou vide et debit est 0, \"0\" ou vide\n} else if ((typeof debit === 'undefined' || debit === null || debit === \"\") && (typeof credit === 'undefined' || credit === null || credit === \"\")) {\n    return null; // Ne pas transmettre le message si debit et credit sont undefined, null ou vide\n}\n    return msg;\n}\n\n// it is tricky, we keep last good message and send it when data is complete, or when we found next good message\nconst filtered = filter(); // for easer instructions\nconst last_message = context.get('last_message');\n\nif (!filtered && context.get('last_message') && msg.parts.index === msg.parts.count - 1) {\n    last_message.complete = true;\n    return last_message;\n}\n\nif (filtered && msg.parts.index === msg.parts.count - 1) {\n    context.set('last_message', filtered)\n    node.send(last_message)\n    msg.complete = true\n    return msg\n}\n\nif (filtered) {\n    context.set('last_message', filtered)\n    return last_message\n}","outputs":1,"timeout":"","noerr":0,"initialize":"","finalize":"","libs":[],"x":710,"y":280,"wires":[["999aed54.e990e","bf1e6320.51cb2","726c27be1f0b435c"]]},{"id":"83d786e7.44c868","type":"watch","z":"487be162.3f1b8","name":"Controller","files":"braincubeAppsData/files/Payroll files from PayFit/Converter deposit","x":180,"y":220,"wires":[["c0a24273.93e8c","2d6ac03d.38e3f"]]},{"id":"bca4efe9.57c36","type":"file in","z":"487be162.3f1b8","name":"Payroll","filename":"","format":"utf8","chunk":false,"sendError":false,"encoding":"none","x":170,"y":340,"wires":[["e3f1352f.019728"]]},{"id":"c0a24273.93e8c","type":"debug","z":"487be162.3f1b8","d":true,"name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":330,"y":220,"wires":[]},{"id":"2d6ac03d.38e3f","type":"change","z":"487be162.3f1b8","name":"Define filename","rules":[{"t":"set","p":"filename","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":200,"y":280,"wires":[["bca4efe9.57c36"]]},{"id":"957b47ce.dc6d28","type":"csv","z":"487be162.3f1b8","name":"","sep":";","hdrin":true,"hdrout":"none","multi":"one","ret":"\\n","temp":"","skip":"0","strings":true,"include_empty_strings":"","include_null_values":"","x":510,"y":280,"wires":[["cb2b1918.600328","ca62b93b.be38b8"]]},{"id":"cb2b1918.600328","type":"debug","z":"487be162.3f1b8","d":true,"name":"Base data","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":660,"y":340,"wires":[]},{"id":"e3f1352f.019728","type":"function","z":"487be162.3f1b8","name":"Redefine msg.file","func":"let filename = msg.file;\nlet transformedFilename = filename.replace('-', '/').replace('.csv', '');\n\nmsg.file = transformedFilename;\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":370,"y":340,"wires":[["957b47ce.dc6d28"]]},{"id":"fd470a8d.6ce038","type":"debug","z":"487be162.3f1b8","d":true,"name":"Converted data","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1100,"y":280,"wires":[]},{"id":"bf1e6320.51cb2","type":"debug","z":"487be162.3f1b8","d":true,"name":"Filtered data","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":930,"y":220,"wires":[]},{"id":"54c64587.dc851c","type":"debug","z":"487be162.3f1b8","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":1390,"y":400,"wires":[]},{"id":"9ea450d1.1807f","type":"join","z":"487be162.3f1b8","name":"","mode":"custom","build":"array","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"","count":"","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":1070,"y":340,"wires":[["ec5c513f.21b3d","d40fda46.f47488"]]},{"id":"ec5c513f.21b3d","type":"function","z":"487be162.3f1b8","name":"Set filename","func":"let filename = msg.file;\nlet transformedFilename = filename.replace('/', '-');\n\nmsg.file = transformedFilename;\nmsg.filename = '/var/lib/braincube-edge/.node-red/braincubeAppsData/files/Payroll files from PayFit/'+ msg.file + '.csv';\n\nreturn msg;","outputs":1,"timeout":"","noerr":0,"initialize":"","finalize":"","libs":[],"x":1090,"y":400,"wires":[["5df58932.d69de8"]]},{"id":"5df58932.d69de8","type":"csv","z":"487be162.3f1b8","name":"","sep":",","hdrin":"","hdrout":"all","multi":"mult","ret":"\\n","temp":"EC_NUMERO,EC_DATE,EC_CODE_JOURNAL,EC_EXERCICE,LI_COMPTE,LI_LIBELLE,LI_DEBIT,LI_CREDIT,LI_DEVISE,LI_PIECE,LI_AFFAIRE","skip":"0","strings":true,"include_empty_strings":true,"include_null_values":true,"x":1250,"y":400,"wires":[["54c64587.dc851c","f5cfa56d.c479e8"]]},{"id":"f5cfa56d.c479e8","type":"file","z":"487be162.3f1b8","d":true,"name":"Converted payrolls","filename":"","appendNewline":true,"createDir":false,"overwriteFile":"true","encoding":"none","x":1430,"y":340,"wires":[[]]},{"id":"28ecef80.f7d3","type":"inject","z":"487be162.3f1b8","name":"","props":[{"p":"payload"},{"p":"file","v":"test_filename","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437300,\"CompteLib\":\"Prevoyance - charges patronales\",\"Credit\":0,\"AxeLib\":\"Operations infrastructure engineering\",\"AxeReference\":\"B000069\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":641400,\"CompteLib\":\"Avantages en nature\",\"Debit\":103.2,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":421000,\"CompteLib\":\"Personnel - remunerations dues\",\"Credit\":0,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437200,\"CompteLib\":\"Caisse de retraite AGIRC-ARRCO - charges salariales\",\"Credit\":234.94,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437300,\"CompteLib\":\"Prevoyance - charges salariales\",\"Credit\":0,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":437300,\"CompteLib\":\"Mutuelle - charges salariales\",\"Credit\":54.98,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":791000,\"CompteLib\":\"Avantages en nature\",\"Credit\":103.2,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":442100,\"CompteLib\":\"Prelevement a la source\",\"Credit\":208.83,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":645210,\"CompteLib\":\"Cotisations prevoyance\",\"Debit\":88.17,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"},{\"EcritureDate\":\"31/05/2024\",\"CompteNum\":631200,\"CompteLib\":\"Taxe d'apprentissage\",\"Debit\":39.71,\"AxeLib\":\"Learning & Collaborative project\",\"AxeReference\":\"B000057\"}]","payloadType":"json","x":510,"y":160,"wires":[["cbc163de1dd1970f"]]},{"id":"6781a04.dfc0e6","type":"debug","z":"487be162.3f1b8","d":true,"name":"Base data","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":660,"y":220,"wires":[]},{"id":"d40fda46.f47488","type":"debug","z":"487be162.3f1b8","name":"Converted data","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1175,"y":340,"wires":[],"l":false},{"id":"4c9098d8.5722c8","type":"comment","z":"487be162.3f1b8","name":"You can use the manuel inject","info":"","x":580,"y":120,"wires":[]},{"id":"726c27be1f0b435c","type":"debug","z":"487be162.3f1b8","name":"debug 6","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":890,"y":460,"wires":[]},{"id":"cbc163de1dd1970f","type":"csv","z":"487be162.3f1b8","name":"","sep":";","hdrin":true,"hdrout":"all","multi":"one","ret":"\\n","temp":"","skip":"0","strings":true,"include_empty_strings":"","include_null_values":"","x":510,"y":220,"wires":[["957b47ce.dc6d28"]]}]
6

This topic was automatically closed 90 days after the last reply. New replies are no longer allowed.