.

loading a web page using c# (optional use of cookies) – SCREEN SCRAPER

This code shows how you can load a web page using c#

 
make sure to include 
using System.Net;
 
string url = "http://www.bitlinkit.com";
    HttpWebRequest wRequest = (HttpWebRequest)WebRequest.Create(url);
 
    //Optional Lines below - for redirects

    wRequest.AllowAutoRedirect = false;
    wRequest.MaximumAutomaticRedirections = 1;
 
    //Optional lines below - for cookies
    static CookieContainer MyCookieContainer = new CookieContainer();    // global static - so that it can remember all cookies
    wRequest.CookieContainer = MyCookieContainer;
//for using a proxy
var webProxy = WebProxy.GetDefaultProxy();
webProxy.UseDefaultCredentials = true;
WebRequest.DefaultWebProxy = webProxy;


 
    WebResponse wResponse = wRequest.GetResponse();
    System.IO.Stream rStream = wResponse.GetResponseStream();
    System.IO.StreamReader sReader = new System.IO.StreamReader(rStream);
 
    //again optional - for cookies
    MyCookieContainer = wRequest.CookieContainer;
 
    string TextFromPage = sReader.ReadToEnd();
 
 
 
 
//---------------------------------------------------------------------------------------- 
 
 
//below lines are optional:   use the following to look at cookies as they come in:
 
            foreach (Cookie cook in wResponse.Cookies)
            {
                Console.WriteLine("Cookie:");
                Console.WriteLine("{0} = {1}", cook.Name, cook.Value);
                Console.WriteLine("Domain: {0}", cook.Domain);
                Console.WriteLine("Path: {0}", cook.Path);
                Console.WriteLine("Port: {0}", cook.Port);
                Console.WriteLine("Secure: {0}", cook.Secure);

                Console.WriteLine("When issued: {0}", cook.TimeStamp);
                Console.WriteLine("Expires: {0} (expired? {1})",
                    cook.Expires, cook.Expired);
                Console.WriteLine("Don't save: {0}", cook.Discard);
                Console.WriteLine("Comment: {0}", cook.Comment);
                Console.WriteLine("Uri for comments: {0}", cook.CommentUri);
                Console.WriteLine("Version: RFC {0}", cook.Version == 1 ? "2109" : "2965");

                // Show the string representation of the cookie.
                Console.WriteLine("String: {0}", cook.ToString());
            }
 

additional code for submitting data (such as login details)

 

 

/// <summary>
/// load a site with upload parameters (such as login)
/// </summary>
public static string getwebsite(string naam, string formparams)
{
    HttpWebRequest wRequest = (HttpWebRequest)WebRequest.Create(naam);

    //die lyn add die cookie waarin die login details gestoor word
    wRequest.CookieContainer = MyCookieContainer;

    WebResponse wResponse;
    if (formparams != null)
    {
        string cookieHeader;
        wRequest.ContentType = "application/x-www-form-urlencoded";
        wRequest.Method = "POST";
        byte[] bytes = Encoding.ASCII.GetBytes(formparams);
        wRequest.ContentLength = bytes.Length;
        using (System.IO.Stream os = wRequest.GetRequestStream())
        {
            os.Write(bytes, 0, bytes.Length);
        }
        wResponse = wRequest.GetResponse();
        cookieHeader = wResponse.Headers["Set-cookie"];
       
        wRequest.Headers.Add("Cookie", cookieHeader);
    }
    else
    {
        wRequest.AllowAutoRedirect = true;
        wResponse = wRequest.GetResponse();
    }
    using (System.IO.Stream rStream = wResponse.GetResponseStream())
    using (System.IO.StreamReader sReader = new System.IO.StreamReader(rStream))
    {
        string returnText = sReader.ReadToEnd();
        return returnText;
    }
   
    HttpWebResponse httpResponse = wResponse as HttpWebResponse;
    var headers = httpResponse.Headers;
    MyCookieContainer.Add(httpResponse.Cookies);

 
} 

where
string formParams = string.Format(“login={0}&password={1}”, “myUname”, “myPword”);

What's your thoughts on this?

*

Protected by WP Anti Spam