第一步 爬取簡(jiǎn)書(shū)推薦用戶
爬取簡(jiǎn)書(shū)推薦用戶阅畴，加入到用戶信息表（第一批用戶）
package com.company;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.*;
public class meizi {
public static void main(String[] args) {
for(int i = 1; i <= 100; i++)
{
Connection connect = Jsoup.connect("http://www.reibang.com/recommendations/users?page=" + i);
try {
// 得到Document對(duì)象
Document document = connect.get();
Elements elements2 = document.select(".wrap");
for(Element element : elements2)
{
Element elements3 = element.select("a").first();
// System.out.println(elements3.attr("abs:href"));
System.out.println(elements3.attr("href").replace("/users/",""));
System.out.println();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
第二步 從推薦用戶出發(fā)，爬取每個(gè)推薦用戶的全部粉絲，加入到用戶信息表（第二批用戶）
package com.company;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
public class meizi2 {
public static void main(String[] args) {
// http://www.reibang.com/users/757b5f9f910b/followers?page=5
//每頁(yè)9條
// 利用Jsoup獲得連接
Connection connect = Jsoup.connect("http://www.reibang.com/users/5ddd9dd5ced9/followers?page=1");
try {
// 得到Document對(duì)象
Document document = connect.get();
// Elements elements = document.select(".wrap");
// System.out.println(elements.get(0).text());
Elements elements2 = document.select(".user-list li");
for(Element element : elements2)
{
Element elements3 = element.select(".info a").first();
System.out.println(elements3.attr("abs:href"));
System.out.println();
// elements3 = element.select(".answer-user-name");
// System.out.println(elements3.text());
//
// elements3 = element.select(".answer-user-tag");
// System.out.println(elements3.text());
//
// elements3 = element.select(".answer-text");
// System.out.println(elements3.text());
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
第三步，爬取每個(gè)簡(jiǎn)書(shū)用戶的全部動(dòng)態(tài)信息
package com.company;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
public class meizi3 {
public static void getDync(int maxid,int pageno)
{
// http://www.reibang.com/users/757b5f9f910b/followers?page=5
//每頁(yè)9條
// 利用Jsoup獲得連接
String query = "?max_id="+maxid+"&page="+pageno;
if(maxid == 0)
{
query = "";
}
Connection connect = Jsoup.connect("http://www.reibang.com/users/c4165d16d0ad/timeline"+query);
try {
// 得到Document對(duì)象
Document document = connect.get();
// Elements elements = document.select(".wrap");
// System.out.println(elements.get(0).text());
Elements elements2 = document.select(".note-list li");
if(elements2.last() == null) {
System.out.println(pageno);
return;
}
String id = elements2.last().id();
maxid = Integer.parseInt(id.split("-")[1]) - 1;
System.out.println(id);
for(Element element : elements2)
{
Elements elements3 = element.select("span[data-type=like_comment]");
if(elements3.size() > 0)
{
System.out.println("喜歡了評(píng)論");
System.out.println(elements3.attr("data-datetime"));
}
elements3 = element.select("span[data-type=comment_note]");
if(elements3.size() > 0)
{
System.out.println("發(fā)表評(píng)論");
System.out.println(element.select(".comment").first().ownText());
System.out.println(element.select("a.title").first().attr("href").replace("/p/",""));
System.out.println(elements3.attr("data-datetime"));
}
elements3 = element.select("span[data-type=like_note]");
if(elements3.size() > 0)
{
System.out.println("喜歡文章");
System.out.println(elements3.attr("data-datetime"));
}
elements3 = element.select("span[data-type=reward_note]");
if(elements3.size() > 0)
{
System.out.println("贊賞文章");
System.out.println(elements3.attr("data-datetime"));
}
elements3 = element.select("span[data-type=share_note]");
if(elements3.size() > 0)
{
System.out.println("發(fā)表文章");
System.out.println(elements3.attr("data-datetime"));
}
elements3 = element.select("span[data-type=like_user]");
if(elements3.size() > 0)
{
System.out.println("關(guān)注作者");
System.out.println(elements3.attr("data-datetime"));
}
elements3 = element.select("span[data-type=like_collection]");
if(elements3.size() > 0)
{
System.out.println("關(guān)注專題");
System.out.println(elements3.attr("data-datetime"));
}
elements3 = element.select("span[data-type=like_notebook]");
if(elements3.size() > 0)
{
System.out.println("關(guān)注文集");
System.out.println(elements3.attr("data-datetime"));
}
}
} catch (IOException e) {
e.printStackTrace();
}
getDync(maxid,++pageno);
}
public static void main(String[] args) {
getDync(0,1);
}
}
