上次做了一個幫公司妹子做了爬蟲，不是很精致窿克，這次公司項目里要用到，于是有做了一番修改毛甲，功能添加了網(wǎng)址圖片采集年叮，下載，線程處理界面網(wǎng)址圖片下載等玻募。
說說思路：首相獲取初始網(wǎng)址的所有內(nèi)容?在初始網(wǎng)址采集圖片?去初始網(wǎng)址采集鏈接?把采集到的鏈接放入隊列?繼續(xù)采集圖片只损，然后繼續(xù)采集鏈接，無限循環(huán)
還是上代碼七咧！
處理網(wǎng)頁內(nèi)容抓取跟網(wǎng)頁網(wǎng)址爬取都做了改進(jìn)跃惫，下面還是大家來看看代碼，有不足之處艾栋，還請之處!
網(wǎng)頁內(nèi)容抓取HtmlCodeRequest,
網(wǎng)頁網(wǎng)址爬取GetHttpLinks爆存，用正則去篩選html中的Links
圖片抓取GetHtmlImageUrlList，用正則去篩選html中的Img
都寫進(jìn)了一個封裝類里面?HttpHelper
//////取得HTML中所有圖片的 URL蝗砾。//////HTML代碼///圖片的URL列表publicstaticstringHtmlCodeRequest(stringUrl)
{if(string.IsNullOrEmpty(Url))
{return"";
}try{//創(chuàng)建一個請求HttpWebRequest httprequst =(HttpWebRequest)WebRequest.Create(Url);//不建立持久性鏈接httprequst.KeepAlive =true;//設(shè)置請求的方法httprequst.Method ="GET";//設(shè)置標(biāo)頭值httprequst.UserAgent ="User-Agent:Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705";
httprequst.Accept="*/*";
httprequst.Headers.Add("Accept-Language","zh-cn,en-us;q=0.5");
httprequst.ServicePoint.Expect100Continue=false;
httprequst.Timeout=5000;
httprequst.AllowAutoRedirect=true;//是否允許302ServicePointManager.DefaultConnectionLimit =30;//獲取響應(yīng)HttpWebResponse webRes =(HttpWebResponse)httprequst.GetResponse();//獲取響應(yīng)的文本流stringcontent =string.Empty;using(System.IO.Stream stream =webRes.GetResponseStream())
{using(System.IO.StreamReader reader =newStreamReader(stream, System.Text.Encoding.GetEncoding("utf-8")))
{
content=reader.ReadToEnd();
}
}//取消請求httprequst.Abort();//返回數(shù)據(jù)內(nèi)容returncontent;
}catch(Exception)
{return"";
}
}//////提取頁面鏈接/////////publicstaticList GetHtmlImageUrlList(stringurl)
{stringhtml =HttpHelper.HtmlCodeRequest(url);if(string.IsNullOrEmpty(html))
{returnnewList();
}//定義正則表達(dá)式用來匹配 img 標(biāo)簽Regex regImg =newRegex(@"]*?\bsrc[\s\t\r\n]*=[\s\t\r\n]*[""']?[\s\t\r\n]*(?[^\s\t\r\n""'<>]*)[^<>]*?/?[\s\t\r\n]*>", RegexOptions.IgnoreCase);//搜索匹配的字符串MatchCollection matches =regImg.Matches(html);
List sUrlList =newList();//取得匹配項列表foreach(Match matchinmatches)
sUrlList.Add(match.Groups["imgUrl"].Value);returnsUrlList;
}//////提取頁面鏈接/////////publicstaticList GetHttpLinks(stringurl)
{//獲取網(wǎng)址內(nèi)容stringhtml =HttpHelper.HtmlCodeRequest(url);if(string.IsNullOrEmpty(html))
{returnnewList();
}//匹配http鏈接conststringpattern2 =@"http(s)?://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?";
Regex r2=newRegex(pattern2, RegexOptions.IgnoreCase);//獲得匹配結(jié)果MatchCollection m2 =r2.Matches(html);
List links =newList();foreach(Match url2inm2)
{if(StringHelper.CheckUrlIsLegal(url2.ToString()) || !StringHelper.IsPureUrl(url2.ToString()) ||links.Contains(url2.ToString()))continue;
links.Add(url2.ToString());
}//匹配href里面的鏈接conststringpattern =@"(?i)]*?href=(['""]?)(?!javascript|__doPostBack)(?[^'""\s*#<>]+)[^>]*>"; ;
Regex r=newRegex(pattern, RegexOptions.IgnoreCase);//獲得匹配結(jié)果MatchCollection m =r.Matches(html);foreach(Match url1inm)
{stringhref1 = url1.Groups["url"].Value;if(!href1.Contains("http"))
{
href1= Global.WebUrl +href1;
}if(!StringHelper.IsPureUrl(href1) || links.Contains(href1))continue;
links.Add(href1);
}returnlinks;
}
這邊下載圖片有個任務(wù)條數(shù)限制先较，限制是200條。如果超過的話線程等待5秒悼粮，這里下載圖片是異步調(diào)用的委托
publicstringDownLoadimg(stringurl)
{if(!string.IsNullOrEmpty(url))
{try{if(!url.Contains("http"))
{
url= Global.WebUrl +url;
}
HttpWebRequest request=(HttpWebRequest)WebRequest.Create(url);
request.Timeout=2000;
request.UserAgent="User-Agent:Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705";//是否允許302request.AllowAutoRedirect =true;
WebResponse response=request.GetResponse();
Stream reader=response.GetResponseStream();//文件名stringaFirstName =Guid.NewGuid().ToString();//擴(kuò)展名stringaLastName = url.Substring(url.LastIndexOf(".") +1, (url.Length - url.LastIndexOf(".") -1));
FileStream writer=newFileStream(Global.FloderUrl + aFirstName +"."+aLastName, FileMode.OpenOrCreate, FileAccess.Write);byte[] buff =newbyte[512];//實際讀取的字節(jié)數(shù)intc =0;while((c = reader.Read(buff,0, buff.Length)) >0)
{
writer.Write(buff,0, c);
}
writer.Close();
writer.Dispose();
reader.Close();
reader.Dispose();
response.Close();return(aFirstName +"."+aLastName);
}catch(Exception)
{return"錯誤：地址"+url;
}
}return"錯誤：地址為空";
}
話不多說闲勺，更多的需要大家自己去改進(jìn)咯！歡迎讀者來與樓主進(jìn)行交流扣猫。如果本文對您有參考價值菜循，歡迎幫博主點下文章下方的推薦，謝謝
下面源碼送上：嘿嘿要分的哦申尤！
http://download.csdn.net/detail/nightmareyan/9627215
