Grab text value from HTML

Frank_R · 11 May 2020 19:47

Hi, im pretty new to node red and trying for a while now to grab some data from an internal webinterface.

I could not get the HTTP request node to work as authentication will fail.
I tried all combinations but no luck. When I use a simple username:pass@ip it does word.

Searching on this forum refers to a solution to use an exec node and a curl command.
curl 'http://username:pass@ip/status.html'

The node trows an error however when I connect the stdout to a debug node I do see the entire contents, so thats fine.

Next step is to grab the interesting value. I tried the html node however I can get it to work. The selector is not able to get to the VAR section. (Or i don;t now how, maybe because of the site scripting?)
I'm also trying the function block with a regex but I need some help with the syntax here.

var parts = msg.payload.match(webdata_now_p[\d.]+));
msg.payload = {
    id: parts[1],
};
return msg;

For reference the entire HTML content:
I;m interested in, for example, var webdata_now_p = "1420"; --> 1420

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<!-- saved from url=(0031)http://192.168.2.15/status.html -->
<html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<style type="text/css">
.in_body
{
	margin-top:0px;
	margin-left:0px;
	margin-right:0px;
	margin-bottom:0px;
	background-color:transparent;
}
.div_c
{
	margin-left:50px;
	margin-right:50px;
	margin-top:50px;
	margin-bottom:50px;
}
.cu
{
	cursor:pointer;
}
.b
{
	font-weight:bold;
}
.lab_5
{
	font-size:16px;
	color:#666666;
	margin-left:-20px;
}
.lab_l2
{
	float:left;
	width:32%;
	color:#666666;
	margin-bottom:-2px;
	font-size:14px;
}
.lab_r2
{
	float:left;
	width:68%;
	color:#666666;
	text-align:right;
	font-size:14px;
}
.cl
{
	clear:left;
}
.line
{
	height:1px;
	background-color:#666666;
	width:100%;
	margin-top:5px;
	margin-bottom:5px;
}
.sp_5
{
	height:5px;
	width:500px;
}
.sp_20
{
	height:20px;
	width:500px;
}
.label
{
	float:left;
	width:50%;
	color:#666666;
	margin-bottom:-2px;
	font-size:14px;
}
.lab_r
{
	float:left;
	width:50%;
	color:#666666;
	text-align:right;
	font-size:14px;
}
.lab_l
{
	float:left;
	width:40%;
	color:#666666;
	margin-bottom:-2px;
	margin-left:10%;
	font-size:14px;
}
.line_l
{
	height:1px;
	background-color:#666666;
	width:450px;
	margin-top:5px;
	margin-bottom:5px;
	margin-left:50px;
}
.sub
{
    display:inline-block;
    width:16px;
    text-align:center;
}
</style>
<script type="text/javascript">
var height=0;function fileText(id,value){if(document.getElementById(id)){document.getElementById(id).innerHTML=value}}function changeFont(){reCon("main_div").style.fontFamily=window.parent.reFont()}function child_getH(){var nh=document.body.offsetHeight+100;if(nh<500||nh==null){nh=500}if(height!=nh){height=nh;window.parent.child_height(height)}}function reCon(id){return document.getElementById(id)}function ready(){try{window.parent.show_ifr()}catch(e){}child_getH()}function show(v){var c=document.getElementById(v);if(c!=null){c.style.display=""}}function hide(v){var c=document.getElementById(v);if(c!=null){c.style.display="none"}};
</script>
<script type="text/javascript">
var webdata_sn = "123456789 ";
var webdata_msvn = "000C";
var webdata_ssvn = "001E";
var webdata_pv_type = "0078";
var webdata_rate_p = "";
var webdata_now_p = "1420";
var webdata_today_e = "24.60";
var webdata_total_e = "2290.0";
var webdata_alarm = "";
var webdata_utime = "0";
var cover_mid = "123456789";
var cover_ver = "MW_08_0501_1.57";
var cover_wmode = "STA";
var cover_ap_ssid = "AP_123456789";
var cover_ap_ip = "10.10.10.10";
var cover_ap_mac = "123456789";
var cover_sta_ssid = "123456789";
var cover_sta_rssi = "100%";
var cover_sta_ip = "10.10.10.10";
var cover_sta_mac = "123456789";
var status_a = "1";
var status_b = "0";
var status_c = "0";

function initPageText(){var list=window.parent.reList("status");fileText("st1",list["t1"]);fileText("st2",list["t2"]);fileText("st3",list["t3"]);for(var i=1;i<=27;i++){if(i!=14){fileText("tx"+i,list[i])}}changeFont();child_getH()}function upfold(v){if(document.getElementById("up_"+v+"_div").style.display=="none"){show("up_"+v+"_div");reCon("p_"+v).innerHTML="-"}else{hide("up_"+v+"_div");reCon("p_"+v).innerHTML="+"}}function init_main_page(){var on=window.parent.reTip("1");var off=window.parent.reTip("2");document.getElementById("cover_mid").innerHTML=cover_mid;document.getElementById("cover_ver").innerHTML=cover_ver;document.getElementById("cover_ap_status").innerHTML=off;document.getElementById("cover_sta_status").innerHTML=off;if(cover_wmode!="STA"){document.getElementById("cover_ap_status").innerHTML=on;document.getElementById("cover_ap_ssid").innerHTML=cover_ap_ssid;document.getElementById("cover_ap_ip").innerHTML=cover_ap_ip;document.getElementById("cover_ap_mac").innerHTML=cover_ap_mac}if(cover_wmode!="AP"){document.getElementById("cover_sta_status").innerHTML=on;document.getElementById("cover_sta_ssid").innerHTML=cover_sta_ssid;document.getElementById("cover_sta_rssi").innerHTML=cover_sta_rssi;document.getElementById("cover_sta_ip").innerHTML=cover_sta_ip;document.getElementById("cover_sta_mac").innerHTML=cover_sta_mac}if(webdata_sn==""){webdata_sn="---"}fileText("webdata_sn",webdata_sn);if(webdata_msvn==""){webdata_msvn="---"}fileText("webdata_msvn",webdata_msvn);if(webdata_ssvn==""){webdata_ssvn="---"}fileText("webdata_ssvn",webdata_ssvn);if(webdata_pv_type==""){webdata_pv_type="---"}fileText("webdata_pv_type",webdata_pv_type);if(webdata_rate_p==""){webdata_rate_p="---"}fileText("webdata_rate_p",webdata_rate_p+" W");if(webdata_now_p==""||webdata_now_p==0){webdata_now_p="---"}fileText("webdata_now_p",webdata_now_p+" W");if(webdata_today_e==""){webdata_today_e="---"}fileText("webdata_today_e",webdata_today_e+" kWh");if(webdata_total_e==""){webdata_total_e="---"}fileText("webdata_total_e",webdata_total_e+" kWh");if(webdata_alarm==""){webdata_alarm="---"}fileText("webdata_alarm",webdata_alarm);if(webdata_utime==""){if(document.getElementById("webdata_sn").innerHTML=="---"){webdata_utime="---"}else{webdata_utime=value+window.parent.reTip("5")}}fileText("webdata_utime",webdata_utime);var st_en=window.parent.reTip("3");var st_dis=window.parent.reTip("4");var st_un=window.parent.reTip("41");if(status_a=="1"){document.getElementById("cover_remote_status_a").innerHTML=st_en}else{if(status_a=="0"){document.getElementById("cover_remote_status_a").innerHTML=st_dis}else{document.getElementById("cover_remote_status_a").innerHTML=st_un}}if(status_b=="1"){document.getElementById("cover_remote_status_b").innerHTML=st_en}else{if(status_b=="0"){document.getElementById("cover_remote_status_b").innerHTML=st_dis}else{document.getElementById("cover_remote_status_b").innerHTML=st_un}}};

</script>
</head>
<body class="in_body" onload="init_main_page();">
	<div class="div_c" id="main_div">
        <div class="lab_5 cu b" onclick="upfold(1);child_getH();"><span class="sub" id="p_1">-</span><span id="st1" style="margin-left:3px"></span></div>
        <div class="sp_5"></div>
        <div id="up_1_div">
        <div class="lab_l2" id="tx1"></div>
                <div class="lab_r2" id="webdata_sn"></div>
        <div class="cl"></div>
        <div class="line"></div>
        <div class="lab_l2" id="tx2"></div>
                <div class="lab_r2" id="webdata_msvn"></div>
        <div class="cl"></div>
        <div class="line"></div>
        <div class="lab_l2" id="tx3"></div>
                <div class="lab_r2" id="webdata_ssvn"></div>
        <div class="cl"></div>
        <div class="line"></div>
        <div class="lab_l2" id="tx4"></div>
                <div class="lab_r2" id="webdata_pv_type"></div>
        <div class="cl"></div>
        <div class="line"></div>
        <div class="lab_l2" id="tx5"></div>
                <div class="lab_r2" id="webdata_rate_p"></div>
        <div class="cl"></div>
        <div class="line"></div>
        <div class="lab_l2" style="color:#666666;font-weight:bold;" id="tx6"></div>
                <div class="lab_r2" id="webdata_now_p" style="color:#666666;font-weight:bold;"></div>
        <div class="cl"></div>
        <div class="line"></div>
        <div class="lab_l2" style="color:#666666;font-weight:bold;" id="tx7"></div>
                <div class="lab_r2" id="webdata_today_e" style="color:#666666;font-weight:bold;"></div>
        <div class="cl"></div>
        <div class="line"></div>
        <div class="lab_l2" style="color:#666666;font-weight:bold;" id="tx8"></div>
                <div class="lab_r2" id="webdata_total_e" style="color:#666666;font-weight:bold;"></div>
        <div class="cl"></div>
        <div class="line"></div>
        <div class="lab_l2" style="color:#666666;font-weight:bold;" id="tx9"></div>
                <div class="lab_r2" id="webdata_alarm" style="color:#666666;font-weight:bold;"></div>
        <div class="cl"></div>
        <div class="line"></div>
        <div class="lab_l2" style="color:#666666;font-weight:bold;" id="tx10"></div>
                <div class="lab_r2" id="webdata_utime" style="color:#666666;font-weight:bold;"></div>
        <div class="cl"></div>
        <div class="line"></div>
        </div>
        <div class="sp_20"></div>
        <div class="lab_5 cu b" onclick="upfold(2);child_getH();"><span class="sub" id="p_2">+</span><span id="st2" style="margin-left:3px"></span></div>
                <div class="sp_5"></div>
                <div id="up_2_div" style="display:none">
                <div class="label" id="tx11"></div>
                <div class="lab_r" id="cover_mid"></div>
                <div class="cl"></div>
                <div class="line"></div>
                <div class="label" id="tx12"></div>
                <div class="lab_r" id="cover_ver"></div>
                <div class="cl"></div>
                <div class="line"></div>
                <div class="label" id="tx13"></div>
                <div class="lab_r" id="cover_ap_status" style="color:#666666;font-weight:bold;"></div>
                <div class="cl"></div>
                <div class="line"></div>
                <div class="lab_l" id="ap_ssid">SSID</div>
                <div class="lab_r" id="cover_ap_ssid"></div>
                <div class="cl"></div>
                <div class="line_l"></div>
                <div class="lab_l" id="tx15"></div>
                <div class="lab_r" id="cover_ap_ip"></div>
                <div class="cl"></div>
                <div class="line_l"></div>
                <div class="lab_l" id="tx16"></div>
                <div class="lab_r" id="cover_ap_mac"></div>
                <div class="cl"></div>
                <div class="line_l"></div>
                <div class="label" id="tx17"></div>
                <div class="lab_r" id="cover_sta_status" style="color:#666666;font-weight:bold;"></div>
                <div class="cl"></div>
                <div class="line"></div>
                <div class="lab_l" id="tx18"></div>
                <div class="lab_r" id="cover_sta_ssid"></div>
                <div class="cl"></div>
                <div class="line_l"></div>
                <div class="lab_l" id="tx19"></div>
                <div class="lab_r" id="cover_sta_rssi"></div>
                <div class="cl"></div>
                <div class="line_l"></div>
                <div class="lab_l" id="tx20"></div>
                <div class="lab_r" id="cover_sta_ip"></div>
                <div class="cl"></div>
                <div class="line_l"></div>
                <div class="lab_l" id="tx21"></div>
                <div class="lab_r" id="cover_sta_mac"></div>
                <div class="cl"></div>
                <div class="line_l"></div>
                </div>
                
                <div class="sp_20"></div>
                <div class="lab_5 cu b" onclick="upfold(3);child_getH();"><span class="sub" id="p_3">+</span><span id="st3" style="margin-left:3px"></span></div>
                <div class="sp_5"></div>
                <div id="up_3_div" style="display:none">
                <div class="label" id="tx25"></div>
                <div class="lab_r" id="cover_remote_status_a"></div>
                <div class="cl"></div>
                <div class="line"></div>
                <div class="label" id="tx26"></div>
                <div class="lab_r" id="cover_remote_status_b"></div>
                <div class="cl"></div>
                <div class="line"></div>
                </div>
    </div>
	<script type="text/javascript">
	    initPageText();
	    ready();
	</script>


</body></html>

Steve-Mcl · 11 May 2020 19:58

Did you use this style in the URL of a http request node? (Without setting basic or digest Auth)

If be surprised if that works with curl but not the request node.

Frank_R · 12 May 2020 11:17

Hi Steve_Mcl,
yes I tried al variations I could think of. This complete URL, with or without additional username/password, basic & digest auth. And the same combinations for the normal URL. On this forums I did found serveral comparing issues were the HTTP nodig could not connect but a direct "call" would.

Anyway, Im a few steps further:
I linked the stdout from the curl command to a function node with the following config

var parts = msg.payload.match(/webdata_now_p\s=\s"(\d+)"/);
msg.payload = {
    id: parts[1],
};
return msg;

This now returns:
{"id":"1400"}

Is this now a value that I can use? Or is it still a text string? I would like to store the 1400 value to home assistant.

Steve-Mcl · 12 May 2020 11:25

Well you wrote msg.payload = { id: parts[1], }; in your code above - so its a variable inside an object called payload which is inside msg!

so in the next node you access it like msg.payload.id (in a function) or payload.id in a switch/change/mustache type node

If you dont understand that - you might want to do a bit JS training and definitely read this

Frank_R · 12 May 2020 18:40

That actually helped me. Thanks.
On step further

Last step is to find a way to send this value to Home Assisstant.

Frank_R · 12 May 2020 19:45

Anybody that can help me to achieve this?
It really hard to connect all the bit ans pieces together for a newbe

Steve-Mcl · 12 May 2020 21:23

Perhaps you need to ask in the HA forum?

Frank_R · 13 May 2020 15:24

One update on this one:

I stumbled upon a www-request node and used this instead of the http-request node and now it works OK !!

The only difference I could notice from the outside is that the www-request node has an option "follwing redirects". However I doubt if that had anything to do with the authentication fail of the http request node. Maybe both nodes work different internally?

Steve-Mcl · 13 May 2020 16:05

I can't see any contrib nodes by that name in the flows library

Which one do you mean?

system · 12 July 2020 16:05

This topic was automatically closed 60 days after the last reply. New replies are no longer allowed.

Topic		Replies	Views
Question about extraction of text on HTML webpage General	7	258	21 October 2022
Extracting data from web page General	5	3053	6 June 2021
Extracting value from HTML \ Javascript General http-request	3	458	16 November 2022
Website Parsing/ Scraping with HTTP request General	10	1007	4 May 2020
Get a value from website General	5	2180	12 May 2019

Grab text value from HTML

Related topics