Remove repeating array objects in the flow

General
1

Hello everyone, I am new with nodeRed and need your help. I have a flow where I get the array of objects using a filter node, but when a new object comes then previous objects also shows in the array. I only want to show latest or newly added object and ignore other objects in the array.

Please also help me, how I can use "change node" for that. Small piece of code based on this flow will be really nice. Thanks.

For testing purpose, You can use the given flow.


[
    {
        "id": "c0db56cb5bb57a0c",
        "type": "debug",
        "z": "3b03cefb5969952e",
        "name": "",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "payload",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 1310,
        "y": 220,
        "wires": []
    },
    {
        "id": "993ea01743adb656",
        "type": "function",
        "z": "3b03cefb5969952e",
        "name": "",
        "func": "msg.method = \"POST\"\nmsg.headers = {'Accept': 'application/json'}\n\nvar dataLoad = msg.payload;\nflow.set(\"dataLoad\", dataLoad);\n\nvar len = dataLoad.length;\n\nlastCreatedDate = msg.payload[0].createdAt;\n\nmsg.lastCreatedDate = lastCreatedDate;\n\nfor (i=0; i<len; i++)\n{\n        msg.payload[i]={\n            \"fields\": {\n                \"Unique Nethunt ID Organization\": msg.payload[i].id,\n                \"Name\": msg.payload[i].fields.Name,\n                \"Link to Nethunt\": msg.payload[i].link,\n                \"Billing Address\": msg.payload[i].fields[\"Billing Address\"],\n                \"tags\": msg.payload[i].fields.Branch\n       }\n    }\n}\n\nreturn msg;\n",
        "outputs": 1,
        "noerr": 0,
        "initialize": "",
        "finalize": "",
        "libs": [],
        "x": 940,
        "y": 220,
        "wires": [
            [
                "3d44e20151b739bd"
            ]
        ]
    },
    {
        "id": "91cb042cc680373a",
        "type": "inject",
        "z": "3b03cefb5969952e",
        "name": "",
        "props": [
            {
                "p": "payload"
            },
            {
                "p": "topic",
                "vt": "str"
            }
        ],
        "repeat": "",
        "crontab": "",
        "once": false,
        "onceDelay": 0.1,
        "topic": "",
        "payload": "[{\"id\":\"6316f3473d2c5949262f33b3\",\"recordId\":\"6316f3473d2c5949262f33b3\",\"createdAt\":\"2022-09-06T07:14:15.460Z\",\"fields\":{\"Name\":\"Test name\",\"Billing Address\":\"address test\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung\",\"Branch\":[\"Heizung und Sanitär\"],\"Business Purpose\":[\"Handwerk\"],\"Products\":[\"6' Lagercontainer\",\"8' Lagercontainer\"],\"Record generated by\":\"Created manually\"},\"link\":\"https://nethunt.com/web/#nethunt\"},{\"id\":\"6315ef4c6aa8992f5209bfbe1\",\"recordId\":\"6315ef4c6aa8992f5209bfbe1\",\"fields\":{\"Name\":\"Test name 1\",\"Billing Address\":\"address test 1\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular 1\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung 1\",\"Branch\":[\"Heizung und Sanitär 1\"],\"Business Purpose\":[\"Handwerk 1\"],\"Products\":[\"6' Lagercontainer 1\",\"8' Lagercontainer 1\"],\"Record generated by\":\"Created manually 1\"},\"link\":\"https://nethunt.com/web/#nethunt 1\"}]",
        "payloadType": "json",
        "x": 790,
        "y": 180,
        "wires": [
            [
                "993ea01743adb656"
            ]
        ]
    },
    {
        "id": "3d44e20151b739bd",
        "type": "rbe",
        "z": "3b03cefb5969952e",
        "name": "",
        "func": "rbe",
        "gap": "",
        "start": "",
        "inout": "out",
        "septopics": true,
        "property": "payload",
        "topi": "topic",
        "x": 1110,
        "y": 220,
        "wires": [
            [
                "c0db56cb5bb57a0c"
            ]
        ]
    },
    {
        "id": "8f690cdbe9d19147",
        "type": "inject",
        "z": "3b03cefb5969952e",
        "name": "",
        "props": [
            {
                "p": "payload"
            },
            {
                "p": "topic",
                "vt": "str"
            }
        ],
        "repeat": "",
        "crontab": "",
        "once": false,
        "onceDelay": 0.1,
        "topic": "",
        "payload": "[{\"id\":\"6316ff796aa8992f520ad92b\",\"recordId\":\"6316ff796aa8992f520ad92b\",\"createdAt\":\"2022-09-06T08:06:17.761Z\",\"fields\":{\"Name\":\"Test name 0\",\"Billing Address\":\"test address 0\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular 0\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung 0\",\"Branch\":[\"Metall- und Stahlbau\"],\"Business Purpose\":[\"Herstellung\"],\"Products\":[\"40' HC 000\"],\"Record generated by\":\"Created manually 0\"},\"link\":\"https://nethunt.com/web/#nethunt/\"},{\"id\":\"6316f3473d2c5949262f33b3\",\"recordId\":\"6316f3473d2c5949262f33b3\",\"createdAt\":\"2022-09-06T07:14:15.460Z\",\"fields\":{\"Name\":\"Test name\",\"Billing Address\":\"address test\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung\",\"Branch\":[\"40' HC 001\"],\"Business Purpose\":[\"Handwerk\"],\"Products\":[\"6' 40' HC 001\",\"8' 40' HC 001\"],\"Record generated by\":\"Created manually\"},\"link\":\"https://nethunt.com/web/#nethunt\"},{\"id\":\"6315ef4c6aa8992f5209bfbe1\",\"recordId\":\"6315ef4c6aa8992f5209bfbe1\",\"createdAt\":\"2022-09-06T07:14:15.460Z\",\"fields\":{\"Name\":\"Test name 1\",\"Billing Address\":\"address test 1\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular 1\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung 1\",\"Branch\":[\"Heizung und Sanitär 1\"],\"Business Purpose\":[\"Handwerk 1\"],\"Products\":[\"6' 40' HC 002\",\"8' 40' HC 002\"],\"Record generated by\":\"Created manually 1\"},\"link\":\"https://nethunt.com/web/#nethunt 1\"}]",
        "payloadType": "json",
        "x": 790,
        "y": 240,
        "wires": [
            [
                "993ea01743adb656"
            ]
        ]
    }
]
2

The idea of using the filter node is not a bad idea, but i think internally it converts the array of objects to a string and does not actually compare/filters each array element.

Perhaps something like this (assuming the recordId) is unique

let input = msg.payload
let stored = flow.get("dataLoad") || []

let tempA = input.reduce((a, { recordId }) => Object.assign(a, { [recordId]: recordId }), {});
let tempB = stored.reduce((a, { recordId }) => Object.assign(a, { [recordId]: recordId }), {});

// merge difference
let result = [...input.filter(({ recordId }) => !tempB[recordId]), ...stored.filter(({ recordId }) => !tempA[recordId])];
msg.payload = result;

flow.set("dataLoad", input)

return msg;

In short, it compares the existing flow variable against the input and outputs the difference - based on recordId.

3

thanks @bakman2 . I tried this solution, but it is not working as I expected. Now it gives me duplicated records if I only update or add one more object in inject node.
The idea is to get new object from same inject node.

4

Hello, Is there any other solution for this? I am really looking for some solution here. :slight_smile:

5

The issue being I don't fully undestand what output you want. If you just want an object made up from the array with the newest date/time, this may work

[{"id":"8f690cdbe9d19147","type":"inject","z":"30af2d3e.d94ea2","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[{\"id\":\"6316ff796aa8992f520ad92b\",\"recordId\":\"6316ff796aa8992f520ad92b\",\"createdAt\":\"2022-09-06T08:06:17.761Z\",\"fields\":{\"Name\":\"Test name 0\",\"Billing Address\":\"test address 0\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular 0\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung 0\",\"Branch\":[\"Metall- und Stahlbau\"],\"Business Purpose\":[\"Herstellung\"],\"Products\":[\"40' HC 000\"],\"Record generated by\":\"Created manually 0\"},\"link\":\"https://nethunt.com/web/#nethunt/\"},{\"id\":\"6316f3473d2c5949262f33b3\",\"recordId\":\"6316f3473d2c5949262f33b3\",\"createdAt\":\"2022-09-06T07:14:15.460Z\",\"fields\":{\"Name\":\"Test name\",\"Billing Address\":\"address test\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung\",\"Branch\":[\"40' HC 001\"],\"Business Purpose\":[\"Handwerk\"],\"Products\":[\"6' 40' HC 001\",\"8' 40' HC 001\"],\"Record generated by\":\"Created manually\"},\"link\":\"https://nethunt.com/web/#nethunt\"},{\"id\":\"6315ef4c6aa8992f5209bfbe1\",\"recordId\":\"6315ef4c6aa8992f5209bfbe1\",\"createdAt\":\"2022-09-06T07:14:15.460Z\",\"fields\":{\"Name\":\"Test name 1\",\"Billing Address\":\"address test 1\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular 1\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung 1\",\"Branch\":[\"Heizung und Sanitär 1\"],\"Business Purpose\":[\"Handwerk 1\"],\"Products\":[\"6' 40' HC 002\",\"8' 40' HC 002\"],\"Record generated by\":\"Created manually 1\"},\"link\":\"https://nethunt.com/web/#nethunt 1\"}]","payloadType":"json","x":190,"y":900,"wires":[["17790163.4c941f"]]},{"id":"17790163.4c941f","type":"change","z":"30af2d3e.d94ea2","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"$$.payload^(>$moment($.createdAt).format())[0].{\t            \"fields\": {\t                \"Unique Nethunt ID Organization\": $.id,\t                \"Name\": $.fields.Name,\t                \"Link to Nethunt\": $.link,\t                \"Billing Address\": $.fields.'Billing Address',\t                \"tags\": $.fields.Branch\t       }}","tot":"jsonata"}],"action":"","property":"","from":"","to":"","reg":false,"x":230,"y":840,"wires":[["c0db56cb5bb57a0c"]]},{"id":"c0db56cb5bb57a0c","type":"debug","z":"30af2d3e.d94ea2","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":570,"y":860,"wires":[]}]

Otherwise you may need to explain more.

6

Hello, @E1cid As you can test my flow I am using a filter node to get only the newly added object. I need something like a filter node but in filter node When a new object comes in array it also prints other objects from the array which is the issue with filter node.

I want something that works like a filter node as my flow is working but I want a workaround for this issue where it also gets older objects in the array.

In your flow it is always printing the last object according to the date. Please help me to get something like filter but not adding older objects from the array.

7

As clear as mud. I think you will need to present examples of original array, then what new incoming array looks like, and then what element of that array you wish not to pass on. It will also help to know what properties decide what element is dropped.

[edit]
I think I may understand better,. One issue was that your 2 element inject, one element was missing a createdAt property.

[{"id":"91cb042cc680373a","type":"inject","z":"30af2d3e.d94ea2","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[{\"id\":\"6316f3473d2c5949262f33b3\",\"recordId\":\"6316f3473d2c5949262f33b3\",\"createdAt\":\"2022-09-06T07:14:15.460Z\",\"fields\":{\"Name\":\"Test name\",\"Billing Address\":\"address test\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung\",\"Branch\":[\"Heizung und Sanitär\"],\"Business Purpose\":[\"Handwerk\"],\"Products\":[\"6' Lagercontainer\",\"8' Lagercontainer\"],\"Record generated by\":\"Created manually\"},\"link\":\"https://nethunt.com/web/#nethunt\"},{\"id\":\"6315ef4c6aa8992f5209bfbe1\",\"recordId\":\"6315ef4c6aa8992f5209bfbe1\",\"createdAt\":\"2022-09-06T07:14:15.460Z\",\"fields\":{\"Name\":\"Test name 1\",\"Billing Address\":\"address test 1\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular 1\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung 1\",\"Branch\":[\"Heizung und Sanitär 1\"],\"Business Purpose\":[\"Handwerk 1\"],\"Products\":[\"6' Lagercontainer 1\",\"8' Lagercontainer 1\"],\"Record generated by\":\"Created manually 1\"},\"link\":\"https://nethunt.com/web/#nethunt 1\"}]","payloadType":"json","x":110,"y":840,"wires":[["993ea01743adb656"]]},{"id":"993ea01743adb656","type":"function","z":"30af2d3e.d94ea2","name":"","func":"const data  = msg.payload;\nconst previous = context.get(\"filter\") || {};\nconst output = [];\nconst filter = {};\n\ndata.forEach((obj) =>{\n    if(previous[obj.id] === undefined && previous[obj.id] != obj.createdAt){\n        filter[obj.id] = obj.createdAt;\n        output.push({\n            \"fields\": {\n                \"Unique Nethunt ID Organization\": obj.id,\n                \"Name\": obj.fields.Name,\n                \"Link to Nethunt\": obj.link,\n                \"Billing Address\": obj.fields[\"Billing Address\"],\n                \"tags\": obj.fields.Branch\n            }\n        })\n    }\n})\ncontext.set(\"filter\",filter);\nmsg.payload = output; \nreturn msg;\n","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":290,"y":860,"wires":[["c0db56cb5bb57a0c"]]},{"id":"8f690cdbe9d19147","type":"inject","z":"30af2d3e.d94ea2","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"[{\"id\":\"6316ff796aa8992f520ad92b\",\"recordId\":\"6316ff796aa8992f520ad92b\",\"createdAt\":\"2022-09-06T08:06:17.761Z\",\"fields\":{\"Name\":\"Test name 0\",\"Billing Address\":\"test address 0\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular 0\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung 0\",\"Branch\":[\"Metall- und Stahlbau\"],\"Business Purpose\":[\"Herstellung\"],\"Products\":[\"40' HC 000\"],\"Record generated by\":\"Created manually 0\"},\"link\":\"https://nethunt.com/web/#nethunt/\"},{\"id\":\"6316f3473d2c5949262f33b3\",\"recordId\":\"6316f3473d2c5949262f33b3\",\"createdAt\":\"2022-09-06T07:14:15.460Z\",\"fields\":{\"Name\":\"Test name\",\"Billing Address\":\"address test\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung\",\"Branch\":[\"40' HC 001\"],\"Business Purpose\":[\"Handwerk\"],\"Products\":[\"6' 40' HC 001\",\"8' 40' HC 001\"],\"Record generated by\":\"Created manually\"},\"link\":\"https://nethunt.com/web/#nethunt\"},{\"id\":\"6315ef4c6aa8992f5209bfbe1\",\"recordId\":\"6315ef4c6aa8992f5209bfbe1\",\"createdAt\":\"2022-09-06T07:14:15.460Z\",\"fields\":{\"Name\":\"Test name 1\",\"Billing Address\":\"address test 1\",\"Country\":\"DE\",\"Contact Source\":\"Kontaktformular 1\",\"Responsible\":[\"fd@mailing.de\"],\"Category\":[\"Kunden\"],\"Processing Status\":\"in Bearbeitung 1\",\"Branch\":[\"Heizung und Sanitär 1\"],\"Business Purpose\":[\"Handwerk 1\"],\"Products\":[\"6' 40' HC 002\",\"8' 40' HC 002\"],\"Record generated by\":\"Created manually 1\"},\"link\":\"https://nethunt.com/web/#nethunt 1\"}]","payloadType":"json","x":110,"y":900,"wires":[["993ea01743adb656"]]},{"id":"c0db56cb5bb57a0c","type":"debug","z":"30af2d3e.d94ea2","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":530,"y":860,"wires":[]}]

If this is not it , Then a clearer explanation with examples will be needed.

8

This topic was automatically closed 60 days after the last reply. New replies are no longer allowed.