webmagic是一個簡單上手的爬蟲框架，提供簡單靈活的API驻龟，只需少量代碼即可實現(xiàn)一個爬蟲

一温眉、引入jar包

        <dependency>
            <groupId>us.codecraft</groupId>
            <artifactId>webmagic-core</artifactId>
            <version>0.5.2</version>
        </dependency>
        <dependency>
            <groupId>us.codecraft</groupId>
            <artifactId>webmagic-extension</artifactId>
            <version>0.5.2</version>
        </dependency>

二、創(chuàng)建爬取類實現(xiàn)PageProcessor

@Component
@Scope("prototype")
public class JcFootballResultProcessor implements PageProcessor{  
    
    private Logger logger = LoggerFactory.getLogger(this.getClass());
    
    private Gson gson = new Gson();
    
    @Autowired
    private MatchResultMapper matchResultMapper;
    
    private Site site = Site.me().setSleepTime(1).setRetryTimes(3);  //爬取時間間隔和重試次數(shù)
    
    public static final String FOOTBALL_URL = "http://info\\.sporttery\\.cn/football/match_result\\.php\\?page=\\d"; 
    private static final String FOOTBALL_URL_REUSLT_INFO = "http://i.sporttery.cn/api/fb_match_info/get_pool_rs/?f_callback=pool_prcess&mid=";
    
    @Override  
    public Site getSite() {  
        return site;  
    }  
      
    @Override
    public void process(Page page) {
        Pattern pattern = Pattern.compile(FOOTBALL_URL);
        Matcher matcher = pattern.matcher(page.getUrl().toString());
        if(!matcher.matches()) {
            return;
        }
//獲取從某個div后的所有tr的元素翁狐，自動轉(zhuǎn)為一個集合
        List<Selectable> trSelectableList = page.getHtml().xpath("http://div[@class=\"match_list\"]").xpath("http://tr").nodes();
        
        for(Selectable trSelectable:trSelectableList) {//遍歷集合类溢，拿到集合里td元素里的html，又自動轉(zhuǎn)為一個結(jié)合，進(jìn)而可以獲取自己需要的網(wǎng)站內(nèi)容文字
            List<String> contentList = trSelectable.xpath("http://td/html()").all();
            if(contentList!=null && !contentList.isEmpty() && contentList.size()>10) {
                String date = contentList.get(0);
                String matchId = contentList.get(1);
                if(!contentList.get(9).equals("已完成") && !contentList.get(9).equals("取消")){
                    logger.error("賽事"+date+":"+matchId+"非完成/取消狀態(tài)闯冷，跳過抓取");
                    continue;
                }
                String tmpid = getMatchUniqueId(matchId);
                String matchUniqueId = "FB" + DateUtil.calWeek(date,tmpid.substring(0, 1)) + tmpid;
  }
......//中間省略
  page.addTargetRequests(page.getHtml().links().regex(FOOTBALL_URL).all());//指定抓取html頁面的符合此正則表達(dá)式的所有鏈接url
}

三砂心、啟動爬取程序

//注入上面的processor
Spider.create(jcFootballResultProcessor).addUrl(FOOTBALL_URL_RESULT).thread(1).run();//addUrl是定義從哪一個頁面開始爬取