Insert flat text and variable into sqlite db

General
1

I run into a new problem :frowning:.
I want to Scrap the data from another site. There are 3 rows(Firma,ArtikelNaam,Prijs) that I want to fill up. The Firma row has to contain "Jaggsalarm" on every column. The ArtikelNaam and Prijs column have to contain the variable data from <span.price-normal> and <div.name>. I already tried to fix the problem myself but I'm really stuck.
image

[{"id":"91fc0577.a9e628","type":"tab","label":"Flow 6","disabled":false,"info":""},{"id":"86f9e54c.949948","type":"inject","z":"91fc0577.a9e628","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":260,"y":380,"wires":[["c31cc9a8.6f2798"]]},{"id":"c31cc9a8.6f2798","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision AxPro","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-axpro-alarmsysteem","tls":"","persist":false,"proxy":"","authType":"","x":470,"y":380,"wires":[["d43af523.ebbb08","fa057bda.4c9468"]]},{"id":"d43af523.ebbb08","type":"html","z":"91fc0577.a9e628","name":"","property":"payload","outproperty":"payload","tag":"span.price-normal","ret":"text","as":"single","x":710,"y":340,"wires":[["905e377d.32ff08"]]},{"id":"fa057bda.4c9468","type":"html","z":"91fc0577.a9e628","name":"","property":"payload","outproperty":"payload","tag":"div.name","ret":"text","as":"single","x":680,"y":420,"wires":[["60736d9.b297a94"]]},{"id":"905e377d.32ff08","type":"function","z":"91fc0577.a9e628","name":"","func":"var ar = [];\nfor(var i = 1; i < msg.payload.length; i += 1) {  // take every second element\n    ar.push(msg.payload[i].split(\"\\n\")[1].trim());\n    msg.test = ar\n}\nmsg.payload = msg.test\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":900,"y":340,"wires":[["60736d9.b297a94"]]},{"id":"60736d9.b297a94","type":"join","z":"91fc0577.a9e628","name":"","mode":"custom","build":"array","property":"payload","propertyType":"msg","key":"payload","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"","count":"","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"num","reduceFixup":"","x":890,"y":420,"wires":[["7d64efd6.fde5c"]]},{"id":"7d64efd6.fde5c","type":"function","z":"91fc0577.a9e628","name":"","func":"let sql = \"INSERT INTO datascraping (Firma,ArtikelNaam,Prijs) VALUES \"\n\nlet ArtikelNaam = msg.payload[0]\nlet Prijs = msg.payload[1]\nlet valuesSql = []\n\n\nArtikelNaam.forEach( (el, index) => {\n    valuesSql.push(`('${el}', '${Prijs[index]}')`)\n})\n\n\n\nmsg.topic = sql + valuesSql.join(\",\")\n\n\nreturn msg","outputs":1,"noerr":0,"initialize":"var myArray = msg.payload;\nvar myArray2 = msg.payload[1];\nmsg.topic = `INSERT INTO datascraping (ArtikelNaam,Prijs) VALUES (\"${myArray.join('\"),(\"')}\")`\nvar topic2 = `(\"${myArray2.join('\"),(\"')}\")`\nvar arrayLength = msg.payload.length;\nfor (var i = 0; i <= arrayLength; i++) {\nmsg.testant = arrayLength\n}\nreturn msg;","finalize":"","x":1040,"y":420,"wires":[["cf7efb68.6ac098","987fc855.328bd8"]]},{"id":"cf7efb68.6ac098","type":"sqlite","z":"91fc0577.a9e628","mydb":"bc3f4a1b.b28ee8","sqlquery":"msg.topic","sql":"","name":"","x":1250,"y":420,"wires":[[]]},{"id":"987fc855.328bd8","type":"debug","z":"91fc0577.a9e628","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":1190,"y":480,"wires":[]},{"id":"bc3f4a1b.b28ee8","type":"sqlitedb","db":"C:\\sqlite\\datascraping.db","mode":"RW"}]

Maybe someone of you guys can help me with this one?

2

Try this .. with some more elegant way of Joining the values from the http-request based on topic

image

[{"id":"86f9e54c.949948","type":"inject","z":"91fc0577.a9e628","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":140,"y":360,"wires":[["c31cc9a8.6f2798"]]},{"id":"c31cc9a8.6f2798","type":"http request","z":"91fc0577.a9e628","name":"Hikvision AxPro","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-axpro-alarmsysteem","tls":"","persist":false,"proxy":"","authType":"","x":320,"y":360,"wires":[["d43af523.ebbb08","fa057bda.4c9468"]]},{"id":"d43af523.ebbb08","type":"html","z":"91fc0577.a9e628","name":"","property":"payload","outproperty":"payload","tag":".price-normal","ret":"text","as":"single","x":490,"y":300,"wires":[["905e377d.32ff08"]]},{"id":"fa057bda.4c9468","type":"html","z":"91fc0577.a9e628","name":"","property":"payload","outproperty":"payload","tag":".name > a","ret":"text","as":"single","x":480,"y":400,"wires":[["1dde846a.2f7914"]]},{"id":"905e377d.32ff08","type":"function","z":"91fc0577.a9e628","name":"","func":"var ar = [];\n\nmsg.payload.forEach( el =>  { \n    ar.push(el.trim().replace(\"€\",\"\"));\n})\n\nmsg.payload = ar\n\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":615,"y":300,"wires":[["3a26af5c.0bdf7"]],"l":false},{"id":"60736d9.b297a94","type":"join","z":"91fc0577.a9e628","name":"","mode":"custom","build":"object","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"","count":"2","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"num","reduceFixup":"","x":770,"y":340,"wires":[["7d64efd6.fde5c","a1c1f5f8.d63578"]]},{"id":"7d64efd6.fde5c","type":"function","z":"91fc0577.a9e628","name":"","func":"let sql = \"INSERT INTO datascraping (Firma,ArtikelNaam,Prijs) VALUES \"\n\nlet ArtikelNaam = msg.payload.name\nlet Prijs = msg.payload.price\nlet valuesSql = []\n\n\nArtikelNaam.forEach( (el, index) => {\n    valuesSql.push(`('Jaggsalarm', '${el}', '${Prijs[index]}')`)\n})\n\n\n\nmsg.topic = sql + valuesSql.join(\",\")\n\n\nreturn msg","outputs":1,"noerr":0,"initialize":"// var myArray = msg.payload;\n// var myArray2 = msg.payload[1];\n// msg.topic = `INSERT INTO datascraping (ArtikelNaam,Prijs) VALUES (\"${myArray.join('\"),(\"')}\")`\n// var topic2 = `(\"${myArray2.join('\"),(\"')}\")`\n// var arrayLength = msg.payload.length;\n// for (var i = 0; i <= arrayLength; i++) {\n// msg.testant = arrayLength\n// }\n// return msg;","finalize":"","x":960,"y":340,"wires":[["987fc855.328bd8"]]},{"id":"987fc855.328bd8","type":"debug","z":"91fc0577.a9e628","name":"2","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":1110,"y":340,"wires":[]},{"id":"1dde846a.2f7914","type":"change","z":"91fc0577.a9e628","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"name","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":675,"y":400,"wires":[["60736d9.b297a94"]],"l":false},{"id":"3a26af5c.0bdf7","type":"change","z":"91fc0577.a9e628","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"price","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":675,"y":300,"wires":[["60736d9.b297a94"]],"l":false},{"id":"a1c1f5f8.d63578","type":"debug","z":"91fc0577.a9e628","name":"1","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":860,"y":260,"wires":[]}]

Not sure if we are picking up the correct items from the web site .. check it because im not sure if you are filtering the items that are not in stock ?!

Adding 'Jaggsalarm' is easy .. during the loop for creating the sql values just add it in as a hardcoded value. see the example

1 Like
3

Thank you for your quick reply!
I used your example and it worked :slight_smile:

I'm pretty new at using node-red but I understand what you did.

I have another question for you :sweat_smile:

When adding the price from the site to the sqlite db the website uses different spans on the same page for the price it uses span.price-normal if it's the normal price and span.price-new if the article is on sale.

I would like to get the normal price and the new price into the Prijs row in the database.

image
image1494×104 4.89 KB

image
image256×801 6.9 KB

image
image509×776 7.4 KB 

[{"id":"91fc0577.a9e628","type":"tab","label":"Flow 6","disabled":false,"info":""},{"id":"cf7efb68.6ac098","type":"sqlite","z":"91fc0577.a9e628","mydb":"bc3f4a1b.b28ee8","sqlquery":"msg.topic","sql":"","name":"","x":1470,"y":820,"wires":[[]]},{"id":"2f3eb64f.4b0d2a","type":"inject","z":"91fc0577.a9e628","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":400,"y":720,"wires":[["769c47ef.502278"]]},{"id":"1aa1d1c5.bf06be","type":"http request","z":"91fc0577.a9e628","name":"Hikvision AxPro","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-axpro-alarmsysteem","tls":"","persist":false,"proxy":"","authType":"","x":540,"y":860,"wires":[[]]},{"id":"20d1662b.7b656a","type":"html","z":"91fc0577.a9e628","name":"","property":"payload","outproperty":"payload","tag":".price-normal","ret":"text","as":"single","x":870,"y":760,"wires":[["f2b92a0.55a90d8"]]},{"id":"6048beb5.0fe01","type":"html","z":"91fc0577.a9e628","name":"","property":"payload","outproperty":"payload","tag":".name > a","ret":"text","as":"single","x":860,"y":860,"wires":[["3889e2b6.c8743e"]]},{"id":"f2b92a0.55a90d8","type":"function","z":"91fc0577.a9e628","name":"","func":"var ar = [];\n\nmsg.payload.forEach( el =>  { \n    ar.push(el.trim().replace(\"€\",\"\"));\n})\n\nmsg.payload = ar\n\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":995,"y":760,"wires":[["f466afe8.7a66d"]],"l":false},{"id":"bfdf31a1.b8754","type":"join","z":"91fc0577.a9e628","name":"","mode":"custom","build":"object","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"","count":"2","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"num","reduceFixup":"","x":1130,"y":840,"wires":[["5b5187d7.efa7e8"]]},{"id":"5b5187d7.efa7e8","type":"function","z":"91fc0577.a9e628","name":"","func":"let sql = \"INSERT INTO datascraping (Firma,ArtikelNaam,Prijs) VALUES \"\n\nlet ArtikelNaam = msg.payload.name\nlet Prijs = msg.payload.price\nlet valuesSql = []\n\n\nArtikelNaam.forEach( (el, index) => {\n    valuesSql.push(`('Jaggsalarm', '${el}', '${Prijs[index]}')`)\n})\n\n\n\nmsg.topic = sql + valuesSql.join(\",\")\n\n\nreturn msg","outputs":1,"noerr":0,"initialize":"// var myArray = msg.payload;\n// var myArray2 = msg.payload[1];\n// msg.topic = `INSERT INTO datascraping (ArtikelNaam,Prijs) VALUES (\"${myArray.join('\"),(\"')}\")`\n// var topic2 = `(\"${myArray2.join('\"),(\"')}\")`\n// var arrayLength = msg.payload.length;\n// for (var i = 0; i <= arrayLength; i++) {\n// msg.testant = arrayLength\n// }\n// return msg;","finalize":"","x":1280,"y":860,"wires":[["60842bcf.594ec4"]]},{"id":"3889e2b6.c8743e","type":"change","z":"91fc0577.a9e628","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"name","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":975,"y":860,"wires":[["bfdf31a1.b8754"]],"l":false},{"id":"f466afe8.7a66d","type":"change","z":"91fc0577.a9e628","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"price","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":1055,"y":760,"wires":[["bfdf31a1.b8754"]],"l":false},{"id":"5b41c990.6c1088","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Alarmsysteem","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-alarmsysteem","tls":"","persist":false,"proxy":"","authType":"","x":590,"y":820,"wires":[[]]},{"id":"484478dd.5410f8","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision AxPro","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-axpro-alarm-accessoires","tls":"","persist":false,"proxy":"","authType":"","x":570,"y":940,"wires":[[]]},{"id":"4570c4ad.a5880c","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision IP camera's 1/7","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-ip-camera","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1020,"wires":[[]]},{"id":"3ea00a67.eac126","type":"comment","z":"91fc0577.a9e628","name":"Alarm accessoires","info":"","x":550,"y":900,"wires":[]},{"id":"b07bf597.a59178","type":"http request","z":"91fc0577.a9e628","name":"Request HD Recorders","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-recorder","tls":"","persist":false,"proxy":"","authType":"","x":570,"y":1820,"wires":[[]]},{"id":"ab445c4b.94859","type":"http request","z":"91fc0577.a9e628","name":"Request HD IP Camerabewaking Set 1/3","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-camera-set","tls":"","persist":false,"proxy":"","authType":"","x":620,"y":1300,"wires":[[]]},{"id":"e686d7fb.8c6048","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision IP camera's 2/7","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-ip-camera?page=2","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1060,"wires":[[]]},{"id":"25207ad2.a08ef6","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision IP camera's 3/7","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-ip-camera?page=3","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1100,"wires":[[]]},{"id":"10601ba8.7d4e84","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision IP camera's 4/7","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-ip-camera?page=4","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1140,"wires":[[]]},{"id":"5c860cd4.3067f4","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision IP camera's 5/7","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-ip-camera?page=5","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1180,"wires":[[]]},{"id":"25b154d3.891a2c","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision IP camera's 6/7","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-ip-camera?page=6","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1220,"wires":[[]]},{"id":"63df7836.6c59c8","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision IP camera's 7/7","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-ip-camera?page=7","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1260,"wires":[[]]},{"id":"82f02b89.b95858","type":"comment","z":"91fc0577.a9e628","name":"Alarmsystemen","info":"","x":540,"y":780,"wires":[]},{"id":"54e19337.b1bdec","type":"comment","z":"91fc0577.a9e628","name":"Camerasystemen","info":"","x":540,"y":980,"wires":[]},{"id":"ab1a0437.7f62f8","type":"http request","z":"91fc0577.a9e628","name":"Request HD IP Camerabewaking Set 2/3","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-camera-set?page=2","tls":"","persist":false,"proxy":"","authType":"","x":620,"y":1340,"wires":[[]]},{"id":"6db4b290.8677fc","type":"http request","z":"91fc0577.a9e628","name":"Request HD IP Camerabewaking Set 3/3","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-camera-set?page=3","tls":"","persist":false,"proxy":"","authType":"","x":620,"y":1380,"wires":[[]]},{"id":"e254d64a.11b298","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision ColorVu","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-colorvu","tls":"","persist":false,"proxy":"","authType":"","x":570,"y":1500,"wires":[[]]},{"id":"4e3f2f84.6fd9e","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Acusense","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-acusense","tls":"","persist":false,"proxy":"","authType":"","x":580,"y":1420,"wires":[[]]},{"id":"b816be29.420cc","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Black Label","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-black-label","tls":"","persist":false,"proxy":"","authType":"","x":590,"y":1460,"wires":[[]]},{"id":"ce0ec470.804ef8","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Gold Label 1/3","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-gold-label","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1540,"wires":[[]]},{"id":"f2f1ff09.e08ad","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Gold Label 2/3","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-gold-label?page=2","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1580,"wires":[[]]},{"id":"40dcc4f9.06e6bc","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Gold Label 3/3","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-gold-label?page=3","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1620,"wires":[[]]},{"id":"b266d800.e50f08","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Gold Label 2.0 1/2","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-gold-label-2-0","tls":"","persist":false,"proxy":"","authType":"","x":610,"y":1660,"wires":[[]]},{"id":"3ce87e3.e5f6482","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Gold Label 2.0 2/2","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-gold-label-2-0?page=2","tls":"","persist":false,"proxy":"","authType":"","x":610,"y":1700,"wires":[[]]},{"id":"cb1b42f5.9b704","type":"http request","z":"91fc0577.a9e628","name":"Request Thermische camera's","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-thermische-camera","tls":"","persist":false,"proxy":"","authType":"","x":590,"y":1740,"wires":[[]]},{"id":"3a3ae09b.c1d5b","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Budget Line","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-budget-line","tls":"","persist":false,"proxy":"","authType":"","x":590,"y":1780,"wires":[[]]},{"id":"6b6c4110.62824","type":"http request","z":"91fc0577.a9e628","name":"Request WiFi IP systemen","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-wifi-ip-systemen","tls":"","persist":false,"proxy":"","authType":"","x":580,"y":1860,"wires":[[]]},{"id":"6e6fe111.78ab6","type":"http request","z":"91fc0577.a9e628","name":"Request Hybride Recorders","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-hybride-recorders","tls":"","persist":false,"proxy":"","authType":"","x":580,"y":1900,"wires":[[]]},{"id":"fd97d01a.82c0b","type":"http request","z":"91fc0577.a9e628","name":"Request 4K Ultra HD Systemen 2/2","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-4k?page=2","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1980,"wires":[[]]},{"id":"51746366.79d09c","type":"http request","z":"91fc0577.a9e628","name":"Request 4K Ultra HD Systemen 1/2","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-4k","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":1940,"wires":[[]]},{"id":"dfdfd1d4.dc2ca","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Darkfighter serie 1/2","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-darkfighter","tls":"","persist":false,"proxy":"","authType":"","x":610,"y":2020,"wires":[[]]},{"id":"bd5568a4.060a28","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Darkfighter serie 2/2","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-darkfighter?page=2","tls":"","persist":false,"proxy":"","authType":"","x":610,"y":2060,"wires":[[]]},{"id":"f2e45132.d70f2","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision High-Risk Solutions","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-high-risk-solutions","tls":"","persist":false,"proxy":"","authType":"","x":610,"y":2100,"wires":[[]]},{"id":"f4240ed5.718e1","type":"http request","z":"91fc0577.a9e628","name":"Request Accessoires 1/4","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-accessoires","tls":"","persist":false,"proxy":"","authType":"","x":570,"y":2140,"wires":[[]]},{"id":"16d1cd2e.43b093","type":"http request","z":"91fc0577.a9e628","name":"Request Accessoires 3/4","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-accessoires?page=3","tls":"","persist":false,"proxy":"","authType":"","x":570,"y":2220,"wires":[[]]},{"id":"4541112e.0cee9","type":"http request","z":"91fc0577.a9e628","name":"Request Accessoires 2/4","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-accessoires?page=2","tls":"","persist":false,"proxy":"","authType":"","x":570,"y":2180,"wires":[[]]},{"id":"f9a3b754.6ce158","type":"http request","z":"91fc0577.a9e628","name":"Request Accessoires 4/4","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-accessoires?page=4","tls":"","persist":false,"proxy":"","authType":"","x":570,"y":2260,"wires":[[]]},{"id":"769c47ef.502278","type":"http request","z":"91fc0577.a9e628","name":"Request Hikvision Intercom","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-intercom","tls":"","persist":false,"proxy":"","authType":"","x":600,"y":620,"wires":[["6048beb5.0fe01","c533f6f3.2ccfc8","20d1662b.7b656a"]]},{"id":"c533f6f3.2ccfc8","type":"html","z":"91fc0577.a9e628","name":"","property":"payload","outproperty":"payload","tag":".price-new","ret":"text","as":"single","x":860,"y":700,"wires":[["2462f9c8.0c12f6"]]},{"id":"2462f9c8.0c12f6","type":"function","z":"91fc0577.a9e628","name":"","func":"var ar = [];\n\nmsg.payload.forEach( el =>  { \n    ar.push(el.trim().replace(\"€\",\"\"));\n})\n\nmsg.payload = ar\n\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":995,"y":700,"wires":[["5e840067.d6da"]],"l":false},{"id":"5e840067.d6da","type":"change","z":"91fc0577.a9e628","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"price","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":1055,"y":700,"wires":[["bfdf31a1.b8754"]],"l":false},{"id":"60842bcf.594ec4","type":"debug","z":"91fc0577.a9e628","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":1300,"y":720,"wires":[]},{"id":"bc3f4a1b.b28ee8","type":"sqlitedb","db":"C:\\sqlite\\datascraping.db","mode":"RW"}]

Kind regard,

Gino

4

I think you can use .price-new, .price-normal as a selector. it should select either one

Updated Flow:

[{"id":"6dde9596.457ea4","type":"inject","z":"42c8e9e3.fd4a18","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":420,"y":540,"wires":[["c48cdfad.c655e8"]]},{"id":"f28b9b31.92141","type":"html","z":"42c8e9e3.fd4a18","name":"","property":"payload","outproperty":"payload","tag":".name > a","ret":"text","as":"single","x":860,"y":580,"wires":[["f97c96bd.1f94d8","64c045a6.43b834"]]},{"id":"ab27aecc.af3c28","type":"join","z":"42c8e9e3.fd4a18","name":"","mode":"custom","build":"object","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":false,"timeout":"","count":"2","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"num","reduceFixup":"","x":1230,"y":540,"wires":[["dd54af12.1b7e"]]},{"id":"dd54af12.1b7e","type":"function","z":"42c8e9e3.fd4a18","name":"","func":"let sql = \"INSERT INTO datascraping (Firma,ArtikelNaam,Prijs) VALUES \"\n\nlet ArtikelNaam = msg.payload.name\nlet Prijs = msg.payload.price\nlet valuesSql = []\n\n\nArtikelNaam.forEach( (el, index) => {\n    valuesSql.push(`('Jaggsalarm', '${el}', '${Prijs[index]}')`)\n})\n\n\n\nmsg.topic = sql + valuesSql.join(\",\")\n\n\nreturn msg","outputs":1,"noerr":0,"initialize":"// var myArray = msg.payload;\n// var myArray2 = msg.payload[1];\n// msg.topic = `INSERT INTO datascraping (ArtikelNaam,Prijs) VALUES (\"${myArray.join('\"),(\"')}\")`\n// var topic2 = `(\"${myArray2.join('\"),(\"')}\")`\n// var arrayLength = msg.payload.length;\n// for (var i = 0; i <= arrayLength; i++) {\n// msg.testant = arrayLength\n// }\n// return msg;","finalize":"","x":1380,"y":540,"wires":[["cad1b484.c6df"]]},{"id":"f97c96bd.1f94d8","type":"change","z":"42c8e9e3.fd4a18","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"name","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":1115,"y":580,"wires":[["ab27aecc.af3c28"]],"l":false},{"id":"c48cdfad.c655e8","type":"http request","z":"42c8e9e3.fd4a18","name":"Request Hikvision Intercom","method":"GET","ret":"txt","paytoqs":"ignore","url":"https://www.jaggsalarm.nl/hikvision-intercom","tls":"","persist":false,"proxy":"","authType":"","x":620,"y":540,"wires":[["f28b9b31.92141","36603f71.27f4f8"]]},{"id":"36603f71.27f4f8","type":"html","z":"42c8e9e3.fd4a18","name":"","property":"payload","outproperty":"payload","tag":".price-new, .price-normal","ret":"text","as":"single","x":910,"y":500,"wires":[["45e1166b.fde738","3a8fea8b.05533e"]]},{"id":"45e1166b.fde738","type":"function","z":"42c8e9e3.fd4a18","name":"","func":"var ar = [];\n\nmsg.payload.forEach( el =>  { \n    ar.push(el.trim().replace(\"€\",\"\"));\n})\n\nmsg.payload = ar\n\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":1075,"y":500,"wires":[["d81d989f.4ff9a"]],"l":false},{"id":"d81d989f.4ff9a","type":"change","z":"42c8e9e3.fd4a18","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"price","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":1115,"y":500,"wires":[["ab27aecc.af3c28"]],"l":false},{"id":"cad1b484.c6df","type":"debug","z":"42c8e9e3.fd4a18","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":1530,"y":540,"wires":[]},{"id":"3a8fea8b.05533e","type":"debug","z":"42c8e9e3.fd4a18","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1100,"y":400,"wires":[]},{"id":"64c045a6.43b834","type":"debug","z":"42c8e9e3.fd4a18","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1040,"y":660,"wires":[]}]

ps. do check if the elements are matching because the more parameters you put the more likely something fails.

5

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.