package com.wonder.spider;

import org.apache.commons.httpclient.Cookie;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.http.HttpEntity;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.junit.Test;

import java.io.IOException;
import java.nio.charset.Charset;

/**
 * Created by xiaohuihui on 2018-05-02
 */
public class HttpClientGet {
public static void main(String[] args) throws Exception {
    //1.創(chuàng)建get請(qǐng)求對(duì)象
    HttpGet get = new HttpGet("http://www.itcast.cn?id=11");
    //2.創(chuàng)建HttpClient對(duì)象
    CloseableHttpClient client = HttpClients.createDefault();
    //3.得到響應(yīng)對(duì)象艾船，接收響應(yīng)的數(shù)據(jù)
    CloseableHttpResponse response = client.execute(get);
    //響應(yīng)行信息
    StatusLine line = response.getStatusLine();
    //響應(yīng)狀態(tài)碼
    int code = line.getStatusCode();
    if (code == 200) {
        //響應(yīng)數(shù)據(jù)
        HttpEntity entity = response.getEntity();
        //把響應(yīng)數(shù)據(jù)轉(zhuǎn)換成字符串
        String html = EntityUtils.toString(entity, Charset.forName("utf-8"));
        System.out.println(html);
    }
    response.close();
}

/**
 * 唱吧首頁(yè)爬取
 * @throws IOException
 */
@Test
public void getChangBa() throws IOException {
    HttpGet get = new HttpGet("http://changba.com/s/QpT3lOar-4DtKbmCFNr_wA?code=011pjohh01Qnsz1aUnfh0mrxhh0pjohq&state=STATE");
    CloseableHttpClient client = HttpClients.createDefault();
    CloseableHttpResponse response = client.execute(get);
    //響應(yīng)行信息
    StatusLine line = response.getStatusLine();
    //響應(yīng)狀態(tài)碼
    int code = line.getStatusCode();
    if (code == 200) {
        //響應(yīng)數(shù)據(jù)
        HttpEntity entity = response.getEntity();
        //把響應(yīng)數(shù)據(jù)轉(zhuǎn)換成字符串
        String html = EntityUtils.toString(entity, Charset.forName("utf-8"));
        System.out.println(html);
    }
    response.close();

}

/**
 * post方法登錄后攜帶cookie訪問(wèn)請(qǐng)求
 */
@Test
public void testLoginPinyougou() {
    String loginUrl = "http://localhost:9102/login";
    // 登陸以后需要訪問(wèn)的頁(yè)面
    String afterLoginUrl = "http://localhost:9102/itemCat/findByParentId.do?parentId=1";

    HttpClient httpClient = new HttpClient();
    // 使用post方式進(jìn)行登錄
    PostMethod postMethod = new PostMethod(loginUrl);
    // 設(shè)置登錄信息
    NameValuePair[] data = {
            new NameValuePair("username", "wangyi"), new NameValuePair("password", "123456"),
            new NameValuePair("m1", "2")
    };

    postMethod.setRequestBody(data);

    // 設(shè)置 HttpClient 接收 Cookie,用與瀏覽器一樣的策略
    httpClient.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    try {
        httpClient.executeMethod(postMethod);
    } catch (IOException e) {
        e.printStackTrace();
    }
    // 獲得登陸后的 Cookie
    Cookie[] cookies = httpClient.getState().getCookies();
    StringBuffer tmpcookies = new StringBuffer();
    for (Cookie c : cookies) {
        tmpcookies.append(c.toString() + ";");
    }

    // 進(jìn)行登錄后的操作
    GetMethod getMethod = new GetMethod(afterLoginUrl);
    // 每次訪問(wèn)需授權(quán)的網(wǎng)址時(shí)需帶上前面的 cookie 作為通行證
    getMethod.setRequestHeader("cookie", tmpcookies.toString());
    // 你還可以通過(guò) PostMethod/GetMethod 設(shè)置更多的請(qǐng)求后數(shù)據(jù)
    // 例如琳要，referer 從哪里來(lái)的叠赦，UA 像搜索引擎都會(huì)表名自己是誰(shuí)，無(wú)良搜索引擎除外
    postMethod.setRequestHeader("Referer", "http://localhost:9102/admin/goods_edit.html");
    postMethod.setRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36");
    try {
        httpClient.executeMethod(getMethod);
        // 打印出返回?cái)?shù)據(jù)捏检，檢驗(yàn)一下是否成功
        String text = getMethod.getResponseBodyAsString();
        System.out.println(text);

    } catch (IOException e) {
        e.printStackTrace();
    }


}

}

圖片.png