使用HttpWebrequest对网站进行模拟操作(附登陆百度demo)

浏览数：33 / 时间：2015年06月09日

这篇文章是在博客园正式的第一篇文章。

不出意外以后将在web的方向发展，前段时间把老早以前做过的webqq机器人重做了一遍，算是对winform的告别吧，巩固了C#方面的知识。

本篇主要介绍了我对模拟http请求方式的介绍和理解。(博客的样式是自己写的，有木有感觉好看呢(•‾??‾?•)??°)

首先看一个get请求

 1  public string GetHtml(string url, string Referer, Encoding encode, bool SaveCookie)
 2         {
 3             HttpWebRequest req = WebRequest.Create(url) as HttpWebRequest;
 4             req.Method = "GET";
 5             req.CookieContainer = this.CookieContainer;
 6             req.Proxy = null;
 7             if (!string.IsNullOrEmpty(Referer))
 8                 req.Referer = Referer;
 9             using (HttpWebResponse hwr = req.GetResponse() as HttpWebResponse)
10             {
11                 if (SaveCookie)
12                 {
13                     this.CookieCollection = hwr.Cookies;
14                     this.CookieContainer.GetCookies(req.RequestUri);
15                 }
16                 using (StreamReader SR = new StreamReader(hwr.GetResponseStream(), encode))
17                 {
18                     return SR.ReadToEnd();
19                 }
20             }
21         }

再看POST

public string PostHtml(string url, string Referer, string data, Encoding encode, bool SaveCookie, int waitTime)
        {
            HttpWebRequest req = WebRequest.Create(url) as HttpWebRequest;
            req.CookieContainer = this.CookieContainer;
            if (waitTime > 0)
            {
                req.Timeout = waitTime;
            }
            req.ContentType = "application/x-www-form-urlencoded";
            req.Method = "POST";
            req.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:30.0) Gecko/20100101 Firefox/30.0";
            req.Proxy = null;
            req.ProtocolVersion = HttpVersion.Version10;
            if (!string.IsNullOrEmpty(Referer))
                req.Referer = Referer;
            byte[] mybyte = Encoding.Default.GetBytes(data);
            req.ContentLength = mybyte.Length;
            using (Stream stream = req.GetRequestStream())
            {
                stream.Write(mybyte, 0, mybyte.Length);
            }
            using (HttpWebResponse hwr = req.GetResponse() as HttpWebResponse)
            {
                if (SaveCookie)
                {
                    this.CookieCollection = hwr.Cookies;
                    this.CookieContainer.GetCookies(req.RequestUri);
                }
                using (StreamReader SR = new StreamReader(hwr.GetResponseStream(), encode))
                {
                    return SR.ReadToEnd();
                }
            }
        }

小结

1.这是我封装的一个httphelp类中的一部分，get貌似不需要那些像什么UserAgent,ContentType之类的东西

2.POST请求中一般要设置ContentType和UserAgent

3.默认请求是GET

如何保持Cookie？

每次请求之后把CookieContainer覆盖即可

HttpWebRequest.CookieContainer和HttpWebResponse.CookieCollection的区别

此处说的是进行一次http请求获得响应之后

CookieContainer是当前域的所有Cookie

CookieCollection是该次请求相关的所有Cookie

什么是referer？

即本次请求的来源，从哪个页面进行http请求的

这里常被服务器用来检测请求是否合法

proxy？

请求的代理，对此我了解不多，忘有人能告之

在请求之前设置proxy=null，可以让请求跳过检查代理，http请求速度立刻上升一个档次！尤其是以前蹭wifi的时候我深有体会

这里是我封装的一个http请求辅助类，大家有兴趣可以参考一下：

 1 public class HttpHelp
 2     {
 3         public CookieContainer CookieContainer { get; set; }
 4 
 5         public CookieCollection CookieCollection { get; set; }
 6 
 7         public HttpHelp()
 8         {
 9             this.CookieCollection = new CookieCollection();
10             this.CookieContainer = new CookieContainer();
11         }
12 
13 
14         public string GetHtml(string url, string Referer, Encoding encode, bool SaveCookie)
15         {
16             HttpWebRequest req = WebRequest.Create(url) as HttpWebRequest;
17             req.Method = "GET";
18             req.CookieContainer = this.CookieContainer;
19             req.Proxy = null;
20             if (!string.IsNullOrEmpty(Referer))
21                 req.Referer = Referer;
22             using (HttpWebResponse hwr = req.GetResponse() as HttpWebResponse)
23             {
24                 if (SaveCookie)
25                 {
26                     this.CookieCollection = hwr.Cookies;
27                     this.CookieContainer.GetCookies(req.RequestUri);
28                 }
29                 using (StreamReader SR = new StreamReader(hwr.GetResponseStream(), encode))
30                 {
31                     return SR.ReadToEnd();
32                 }
33             }
34         }
35 
36         public static string GetHtml(string url, string Referer, Encoding encode)
37         {
38             return new HttpHelp().GetHtml(url, Referer, encode, false);
39         }
40 
41         public string PostHtml(string url, string Referer, string data, Encoding encode, bool SaveCookie)
42         {
43             return Poll(url, Referer, data, encode, SaveCookie, 0);
44         }
45 
46         public string Poll(string url, string Referer, string data, Encoding encode, bool SaveCookie, int waitTime)
47         {
48             HttpWebRequest req = WebRequest.Create(url) as HttpWebRequest;
49             req.CookieContainer = this.CookieContainer;
50             if (waitTime > 0)
51             {
52                 req.Timeout = waitTime;
53             }
54             req.ContentType = "application/x-www-form-urlencoded";
55             req.Method = "POST";
56             req.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:30.0) Gecko/20100101 Firefox/30.0";
57             req.Proxy = null;
58             req.ProtocolVersion = HttpVersion.Version10;
59             if (!string.IsNullOrEmpty(Referer))
60                 req.Referer = Referer;
61             byte[] mybyte = Encoding.Default.GetBytes(data);
62             req.ContentLength = mybyte.Length;
63             using (Stream stream = req.GetRequestStream())
64             {
65                 stream.Write(mybyte, 0, mybyte.Length);
66             }
67             using (HttpWebResponse hwr = req.GetResponse() as HttpWebResponse)
68             {
69                 if (SaveCookie)
70                 {
71                     this.CookieCollection = hwr.Cookies;
72                     this.CookieContainer.GetCookies(req.RequestUri);
73                 }
74                 using (StreamReader SR = new StreamReader(hwr.GetResponseStream(), encode))
75                 {
76                     return SR.ReadToEnd();
77                 }
78             }
79         }
80 
81         public Image getImage(string url, string Referer, bool SaveCookie)
82         {
83             HttpWebRequest req = WebRequest.Create(url) as HttpWebRequest;
84             req.Method = "GET";
85             req.CookieContainer = this.CookieContainer;
86             req.Proxy = null;
87             if (!string.IsNullOrEmpty(Referer))
88                 req.Referer = Referer;
89             using (HttpWebResponse hwr = req.GetResponse() as HttpWebResponse)
90             {
91                 if (SaveCookie)
92                 {
93                     this.CookieCollection = hwr.Cookies;
94                     this.CookieContainer.GetCookies(req.RequestUri);
95                 }
96                 return Image.FromStream(hwr.GetResponseStream());
97             }
98         }
99     }