Help to login this page... I want to do web scraping

I did find this.... How to get started with nbrowser? To login into router webpage

Looks like nbrowser does not work anymore... dependencies used may be dead

Have you checked the issues log on github:

If this is a new issue, create a new issue in that github repository.

1 Like

Yeah... He himself have mentioned that nbrowser is end of life...
These are the payload I get when I login into router page @janvda

You can of course also share some details here about the issue you are facing with nbrowser.

1 Like

I know How to do it, I had a similar problem.
When you log in using the browser a token is generated and sent with the log in session. that token must be used on the cookies to do Http post. the problem is that the token changes when you try to log in, you need to capture the token, create a new cookie session and use the previous token to do the log in, let me check how I solved it and I will post how to do it

1 Like

I did it using exec node on windows with curl commands, I started viewing the Network traffic in the browser using inspect mode and since you can copy each POST/GET requests as curl commands and use CMD to execute the same curl commands was easy to troubleshoot.

Once I got the correct CURL commands that worked on a terminal, I needed to automate the same actions using exec node. you need to know how the cookie commands works at this point I don't remember very well but my flow just works for me:

login inject makes the first login attempt, then scrapes the generated token (you will need to figure out where to get the token because is hidden in the HTML, I carefully inspected the payloads resulting from my first login and the nodes that I am using probably won't work for your application) then I use a function node to generate the next command which takes the token, then adds the login credentials and creates a cookie file and stores that token and logged in session and send the new curl command to exec node. once that the logged in session was created and stored on a cookie file, I make a second function to generate a third CURL command to use on another exec to access the page that I wanted to scrap /moni_1 (this is the part that I execute an x given amount of time to scrap the site)

if for some reason the session expires (A computer reset maybe) you need to do the previous two steps again

[{"id":"7f5cb373.31e45c","type":"exec","z":"4c4e122d.50fd5c","command":"","addpay":"payload","append":"","useSpawn":"false","timer":"","oldrc":false,"name":"","x":450,"y":300,"wires":[["4ced2e5a.8bbbb","97a9e9a1.99bf38"],[],[]]},{"id":"4ced2e5a.8bbbb","type":"debug","z":"4c4e122d.50fd5c","name":"raw","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":590,"y":260,"wires":[]},{"id":"cab9edf2.46183","type":"debug","z":"4c4e122d.50fd5c","name":"array","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1130,"y":260,"wires":[]},{"id":"d7f76ca3.6fe0b","type":"html","z":"4c4e122d.50fd5c","name":"","property":"payload","outproperty":"payload","tag":".wg_output_text","ret":"html","as":"single","x":1020,"y":420,"wires":[["ca5a97ae.5dd638"]]},{"id":"ca5a97ae.5dd638","type":"debug","z":"4c4e122d.50fd5c","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":1190,"y":420,"wires":[]},{"id":"8d6be921.fecda8","type":"inject","z":"4c4e122d.50fd5c","name":"login","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"curl -j -L -c /Users/fcastro/login_cookie.txt -H \"Accept-Language: en-US,en;q=0.9,es;q=0.8\" http://192.168.1.14/login","payloadType":"str","x":310,"y":300,"wires":[["7f5cb373.31e45c"]]},{"id":"97a9e9a1.99bf38","type":"html","z":"4c4e122d.50fd5c","name":"","property":"payload","outproperty":"payload","tag":"script[type]","ret":"text","as":"multi","x":610,"y":300,"wires":[["c637a9ba.251148"]]},{"id":"c637a9ba.251148","type":"split","z":"4c4e122d.50fd5c","name":"","splt":";","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":true,"addname":"","x":750,"y":300,"wires":[["12bf4a2.6a0c8b6"]]},{"id":"12bf4a2.6a0c8b6","type":"switch","z":"4c4e122d.50fd5c","name":"","property":"payload","propertyType":"msg","rules":[{"t":"cont","v":"_csrf_token","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":870,"y":300,"wires":[["1819d8e.39c4927"]]},{"id":"1819d8e.39c4927","type":"function","z":"4c4e122d.50fd5c","name":"","func":"\nvar regex = /\\'(.*?)\\'/;\nvar strToMatch = msg.payload;\nvar matched = regex.exec(strToMatch);\n    var curlCMD =  `curl -c /Users/winuser/login_cookie.txt -b /Users/winuser/login_cookie.txt -d \"username=THISISTHEUSERNAME\" -d \"password=THISISTHEPASSWORD\" -d \"_csrf_token=${matched[1]}\" -H \"Accept-Language: en-US,en;q=0.9,es;q=0.8\" http://192.168.1.14/login`\n//msg.payload=matched[1]\nmsg={}\nmsg.payload=curlCMD\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":1000,"y":300,"wires":[["cab9edf2.46183","f42532f8.a3b82"]]},{"id":"ea1c3bd5.828918","type":"link out","z":"4c4e122d.50fd5c","name":"Login","links":["4a65d7c2.c88f28"],"x":1235,"y":300,"wires":[]},{"id":"dc338e6f.8087f","type":"comment","z":"4c4e122d.50fd5c","name":"Request Token","info":"","x":420,"y":260,"wires":[]},{"id":"775f7683.6a50b8","type":"comment","z":"4c4e122d.50fd5c","name":"use Token and form /login CURL cmd","info":"","x":910,"y":260,"wires":[]},{"id":"4a65d7c2.c88f28","type":"link in","z":"4c4e122d.50fd5c","name":"","links":["ea1c3bd5.828918"],"x":315,"y":360,"wires":[["8d61fcea.7e619"]]},{"id":"8d61fcea.7e619","type":"exec","z":"4c4e122d.50fd5c","command":"","addpay":"payload","append":"","useSpawn":"false","timer":"","oldrc":false,"name":"","x":430,"y":360,"wires":[["fa685e75.17909","573b5fff.420cf"],[],[]]},{"id":"fa685e75.17909","type":"debug","z":"4c4e122d.50fd5c","name":"raw","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","targetType":"full","statusVal":"","statusType":"auto","x":590,"y":360,"wires":[]},{"id":"7206c60a.acb788","type":"exec","z":"4c4e122d.50fd5c","command":"","addpay":"payload","append":"","useSpawn":"false","timer":"","oldrc":false,"name":"","x":870,"y":420,"wires":[["d7f76ca3.6fe0b"],[],[]]},{"id":"f42532f8.a3b82","type":"delay","z":"4c4e122d.50fd5c","name":"","pauseType":"delay","timeout":"1","timeoutUnits":"seconds","rate":"1","nbRateUnits":"1","rateUnits":"second","randomFirst":"1","randomLast":"5","randomUnits":"seconds","drop":false,"x":1140,"y":300,"wires":[["ea1c3bd5.828918"]]},{"id":"4c6d9795.4fd2f8","type":"delay","z":"4c4e122d.50fd5c","name":"","pauseType":"delay","timeout":"1","timeoutUnits":"seconds","rate":"1","nbRateUnits":"1","rateUnits":"second","randomFirst":"1","randomLast":"5","randomUnits":"seconds","drop":false,"x":740,"y":420,"wires":[["7206c60a.acb788"]]},{"id":"7265d394.e7efbc","type":"inject","z":"4c4e122d.50fd5c","name":"logout","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":230,"y":500,"wires":[["d00ddf59.ef48"]]},{"id":"c7d1edf3.eb9be","type":"function","z":"4c4e122d.50fd5c","name":"","func":"var curlCMD = `curl -c /Users/winuser/login_cookie.txt -b /Users/winuser/login_cookie.txt -d \"_csrf_token=${msg.payload}\" -H \"Accept-Language: en-US,en;q=0.9,es;q=0.8\" http://192.168.1.14/logout`\nmsg = {}\nmsg.payload = curlCMD\nreturn msg;\n\n\n","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":1140,"y":500,"wires":[["bebff6f5.4b8df8"]]},{"id":"bb03a61d.b48178","type":"debug","z":"4c4e122d.50fd5c","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":1430,"y":500,"wires":[]},{"id":"d00ddf59.ef48","type":"file in","z":"4c4e122d.50fd5c","name":"","filename":"/Users/winuser/login_cookie.txt","format":"utf8","chunk":false,"sendError":false,"encoding":"none","x":450,"y":500,"wires":[["2b386281.41247e"]]},{"id":"2b386281.41247e","type":"split","z":"4c4e122d.50fd5c","name":"","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":650,"y":500,"wires":[["8beb1ae2.31ca38"]]},{"id":"8beb1ae2.31ca38","type":"switch","z":"4c4e122d.50fd5c","name":"","property":"payload","propertyType":"msg","rules":[{"t":"cont","v":"_csrf_token","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":770,"y":500,"wires":[["239cb507.7b9d9a"]]},{"id":"239cb507.7b9d9a","type":"split","z":"4c4e122d.50fd5c","name":"","splt":"_csrf_token\t","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":890,"y":500,"wires":[["3c711a24.61beb6"]]},{"id":"573b5fff.420cf","type":"function","z":"4c4e122d.50fd5c","name":"","func":"msg.payload=`curl \"http://192.168.1.14/moni_1\" ^   -H \"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\" ^   -H \"Accept-Language: en-US,en;q=0.9,es;q=0.8\" ^   -H \"Cache-Control: max-age=0\" ^   -H \"Connection: keep-alive\" ^   -H \"Referer: http://192.168.1.14/startpage\" ^   -H \"Upgrade-Insecure-Requests: 1\" ^   -H \"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36 Edg/102.0.1245.39\" ^   --insecure ^   -c /Users/winuser/login_cookie.txt -b /Users/winuser/login_cookie.txt`\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":600,"y":420,"wires":[["4c6d9795.4fd2f8"]]},{"id":"9301edcf.b978f","type":"comment","z":"4c4e122d.50fd5c","name":"Prepare /moni_1 CURL CMD","info":"","x":820,"y":380,"wires":[]},{"id":"bebff6f5.4b8df8","type":"exec","z":"4c4e122d.50fd5c","command":"","addpay":"payload","append":"","useSpawn":"false","timer":"","oldrc":false,"name":"","x":1270,"y":500,"wires":[["bb03a61d.b48178"],[],[]]},{"id":"3c711a24.61beb6","type":"switch","z":"4c4e122d.50fd5c","name":"","property":"payload","propertyType":"msg","rules":[{"t":"cont","v":"#HttpOnly","vt":"str"},{"t":"else"}],"checkall":"true","repair":false,"outputs":2,"x":1010,"y":500,"wires":[[],["c7d1edf3.eb9be"]]},{"id":"ae445059.67b77","type":"link in","z":"4c4e122d.50fd5c","name":"statusRequest","links":[],"x":395,"y":420,"wires":[["573b5fff.420cf"]]},{"id":"465871a9.948d5","type":"inject","z":"4c4e122d.50fd5c","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":240,"y":420,"wires":[["573b5fff.420cf"]]}]
1 Like

thanks for responding but this is not working for my router...is their can way i can contact yu (like discord)?

@fercasjr are you still their? I need help, I cant find exact curl command?