how to get inspect element code using c#

c# html-agility-pack

Question

I want to get text from URL but text is not shown in source code. I can see it only in inspect element . Is there anyway, in C# to get the contents of Inspect element of the page. I try htmlagilitypack with c# but give null exception.

Class1 class1obj = new Class1(); 
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);  
request.Method = "GET"; 
HttpWebResponse response = (HttpWebResponse)request.GetResponse(); 
if (response.StatusCode == HttpStatusCode.OK) 
{ 
    Stream receiveStream = response.GetResponseStream(); 
    StreamReader readStream = null; 
    if (response.CharacterSet == null)
        readStream = new StreamReader(receiveStream); 
    else
        readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet)); 
    class1obj.Data1 = readStream.ReadToEnd(); 
    Console.WriteLine(class1obj.Data1);
    readStream.Close();
 }

 HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
 doc.LoadHtml(class1obj.Data1);
 // HtmlNodeCollection tables2 =     doc.DocumentNode.SelectNodes(".//div[@class='tv_ch_detila h_scroll_detail']");
 HtmlNode rows2 = doc.DocumentNode.SelectSingleNode(".//div[@class='sub_text']");
 Console.WriteLine(rows2.InnerText);

Popular Answer

It's kind of encrypted data that is being loaded dynamically after the page loads. The Url getting the said data is:

http://www.whatsonindia.com/appi/user?channelgenre=all&context=applicationname%3Dsourcebits%3Bheadendid%3D0&dateselected=0&mode=getTVGuideInfo&pageno=1&responseformat=json&responselanguage=English&starthour=0&totalhrdata=24&userid=-1

It contains a "data" and a "pki" node in JSON format. You can't just reply on waiting for the page being completely loaded.

Instead you can just make a request to the given url and decrypt the text. This is the only way that I can assume here. The unreliable way is to load the url in a Web Browser control and wait for few seconds (say 5-10) assuming that all the contents are loaded and then parse the DocumentText or web browser control.

I managed to get this JSON data for tv-programmes:

{
"gettvguidewithgenre": {
    "tvguideprogrammewithgenrelist": [
      {
        "programmeid": "10000000934920000",
        "starttime": "2014-02-25T00:00:00+05:30",
        "endtime": "2014-02-25T01:40:00+05:30",
        "channelid": "10000000000110000",
        "programmename": "Ghatak",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Action",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549824121",
        "starttime": "2014-02-25T01:40:00+05:30",
        "endtime": "2014-02-25T03:50:00+05:30",
        "channelid": "10000000000110000",
        "programmename": "Time Limit-36 Ghante",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Drama",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549816018",
        "starttime": "2014-02-25T03:50:00+05:30",
        "endtime": "2014-02-25T05:40:00+05:30",
        "channelid": "10000000000110000",
        "programmename": "Volcano High",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Action",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549673440",
        "starttime": "2014-02-25T05:40:00+05:30",
        "endtime": "2014-02-25T08:10:00+05:30",
        "channelid": "10000000000110000",
        "programmename": "Maa Beti",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Drama",
        "Isrecommended": "1"
      },
      {
        "programmeid": "10000001008740000",
        "starttime": "2014-02-25T08:10:00+05:30",
        "endtime": "2014-02-25T11:05:00+05:30",
        "channelid": "10000000000110000",
        "programmename": "Dus",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Action",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549644318",
        "starttime": "2014-02-25T11:05:00+05:30",
        "endtime": "2014-02-25T13:55:00+05:30",
        "channelid": "10000000000110000",
        "programmename": "Money Hai Toh Honey Hai",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Comedy",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549646489",
        "starttime": "2014-02-25T13:55:00+05:30",
        "endtime": "2014-02-25T17:15:00+05:30",
        "channelid": "10000000000110000",
        "programmename": "Farz",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Drama",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549709799",
        "starttime": "2014-02-25T17:15:00+05:30",
        "endtime": "2014-02-25T20:00:00+05:30",
        "channelid": "10000000000110000",
        "programmename": "Action Replayy",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Comedy",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549664553",
        "starttime": "2014-02-25T20:00:00+05:30",
        "endtime": "2014-02-25T23:25:00+05:30",
        "channelid": "10000000000110000",
        "programmename": "Indian",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Action",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549781987",
        "starttime": "2014-02-25T23:25:00+05:30",
        "endtime": "2014-02-26T00:00:00+05:30",
        "channelid": "10000000000110000",
        "programmename": "Narsimha - Man Of Power",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Drama",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549741259",
        "starttime": "2014-02-25T00:00:00+05:30",
        "endtime": "2014-02-25T02:00:00+05:30",
        "channelid": "10000000000540000",
        "programmename": "The Dirty Picture",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Drama",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549867681",
        "starttime": "2014-02-25T02:00:00+05:30",
        "endtime": "2014-02-25T02:40:00+05:30",
        "channelid": "10000000000540000",
        "programmename": "Extraa Shots Special",
        "m2ecategoryname": "TV Show",
        "m2esubcategoryname": "Entertainment",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549867681",
        "starttime": "2014-02-25T02:40:00+05:30",
        "endtime": "2014-02-25T03:00:00+05:30",
        "channelid": "10000000000540000",
        "programmename": "Extraa Shots Special",
        "m2ecategoryname": "TV Show",
        "m2esubcategoryname": "Entertainment",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549867681",
        "starttime": "2014-02-25T03:00:00+05:30",
        "endtime": "2014-02-25T03:25:00+05:30",
        "channelid": "10000000000540000",
        "programmename": "Extraa Shots Special",
        "m2ecategoryname": "TV Show",
        "m2esubcategoryname": "Entertainment",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549867681",
        "starttime": "2014-02-25T03:25:00+05:30",
        "endtime": "2014-02-25T03:40:00+05:30",
        "channelid": "10000000000540000",
        "programmename": "Extraa Shots Special",
        "m2ecategoryname": "TV Show",
        "m2esubcategoryname": "Entertainment",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549867681",
        "starttime": "2014-02-25T03:40:00+05:30",
        "endtime": "2014-02-25T04:00:00+05:30",
        "channelid": "10000000000540000",
        "programmename": "Extraa Shots Special",
        "m2ecategoryname": "TV Show",
        "m2esubcategoryname": "Entertainment",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549867681",
        "starttime": "2014-02-25T04:00:00+05:30",
        "endtime": "2014-02-25T04:25:00+05:30",
        "channelid": "10000000000540000",
        "programmename": "Extraa Shots Special",
        "m2ecategoryname": "TV Show",
        "m2esubcategoryname": "Entertainment",
        "Isrecommended": "1"
      },
      {
        "programmeid": "10000001078790000",
        "starttime": "2014-02-25T18:08:00+05:30",
        "endtime": "2014-02-25T20:30:00+05:30",
        "channelid": "10000000000210000",
        "programmename": "Aadmi",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Action",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549651842",
        "starttime": "2014-02-25T20:30:00+05:30",
        "endtime": "2014-02-25T23:16:00+05:30",
        "channelid": "10000000000210000",
        "programmename": "Dum",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Action",
        "Isrecommended": "1"
      },
      {
        "programmeid": "30000000549828153",
        "starttime": "2014-02-25T23:16:00+05:30",
        "endtime": "2014-02-26T00:00:00+05:30",
        "channelid": "10000000000210000",
        "programmename": "Aakhri Baazi",
        "m2ecategoryname": "Film",
        "m2esubcategoryname": "Romance",
        "Isrecommended": "1"
      }
    ],
    "channelinfo": [
      {
        "ChannelID": "10000000000110000",
        "ChannelName": "STAR GOLD",
        "LCN": "0",
        "Operator_Id": "2645",
        "logofileurl": "http://images.whatsonindia.com/WhatsOnTv/Images/applicationchannellogo/10000000000110000.png",
        "ischannelfavorite": "false"
      },
      {
        "ChannelID": "10000000000540000",
        "ChannelName": "MAX",
        "LCN": "0",
        "Operator_Id": "2645",
        "logofileurl": "http://images.whatsonindia.com/WhatsOnTv/Images/applicationchannellogo/10000000000540000.png",
        "ischannelfavorite": "false"
      },
      {
        "ChannelID": "10000000000620000",
        "ChannelName": "TEN SPORTS",
        "LCN": "0",
        "Operator_Id": "2645",
        "logofileurl": "http://images.whatsonindia.com/WhatsOnTv/Images/applicationchannellogo/10000000000620000.png",
        "ischannelfavorite": "false"
      },
      {
        "ChannelID": "10000000000700000",
        "ChannelName": "HBO",
        "LCN": "0",
        "Operator_Id": "2645",
        "logofileurl": "http://images.whatsonindia.com/WhatsOnTv/Images/applicationchannellogo/10000000000700000.png",
        "ischannelfavorite": "false"
      },
      {
        "ChannelID": "10000000000100000",
        "ChannelName": "STAR MOVIES",
        "LCN": "0",
        "Operator_Id": "2645",
        "logofileurl": "http://images.whatsonindia.com/WhatsOnTv/Images/applicationchannellogo/10000000000100000.png",
        "ischannelfavorite": "false"
      },
      {
        "ChannelID": "10000000000080000",
        "ChannelName": "STAR PLUS",
        "LCN": "0",
        "Operator_Id": "2645",
        "logofileurl": "http://images.whatsonindia.com/WhatsOnTv/Images/applicationchannellogo/10000000000080000.png",
        "ischannelfavorite": "false"
      },
      {
        "ChannelID": "10000000000210000",
        "ChannelName": "ZEE CINEMA",
        "LCN": "0",
        "Operator_Id": "2645",
        "logofileurl": "http://images.whatsonindia.com/WhatsOnTv/Images/applicationchannellogo/10000000000210000.png",
        "ischannelfavorite": "false"
      }
    ]
  }
}

It uses CryptoJS for encryption and decryption.




Licensed under: CC-BY-SA with attribution
Not affiliated with Stack Overflow
Is this KB legal? Yes, learn why
Licensed under: CC-BY-SA with attribution
Not affiliated with Stack Overflow
Is this KB legal? Yes, learn why