我們學(xué)校的所有通知是在同一個網(wǎng)站上發(fā)布的中燥，每天都會有很多條。我這個人比較懶塘偎，不喜歡經(jīng)常翻看校內(nèi)通知網(wǎng)疗涉，所以經(jīng)常會錯過某些重要的校內(nèi)通知。所以最近突發(fā)奇想吟秩，我干脆做一個爬蟲幫我每天看看都有哪些對我有用的通知吧咱扣。

const utils = require('../../myUtils')
const Crawler = require("crawler")
const oa = require('../../mongodb/jlu.edu.cn/oa')

const rootUrl = 'https://oa.jlu.edu.cn/defaultroot/'
const homepageUrl = 'https://oa.jlu.edu.cn/defaultroot/PortalInformation!jldxList.action'
let numOfNotifications = 0
let numOfPages = 0

function getPageUrl(num)
{
    return `${homepageUrl}?1=1&startPage=${num}`;
}

function handlePage(error, res, done)
{
    if(error)
    {
        console.log(error)
        return
    }
    else
    {
        const $ = res.$
        const $items = $("#itemContainer>div")
        $items.each(async function(index, element)
        {
            const $item = $.load($(element).html())
            const notification = {}
            notification.href = rootUrl + $item(".font14").attr("href")
            notification.title = $item(".font14").text().replace('[置頂]', '')
            notification.releaser = $item(".column").text()
            notification.date = $item(".time").text().replace(/今天*/, utils.date())
            await handleNotification(notification)
            numOfNotifications ++
        })
        numOfPages ++
        console.log(`crawled ${numOfPages} pages, ${numOfNotifications} notifications`)
    }
    done()
}

async function handleNotification(notification)
{
    const _notification = await oa.findOneNotification(notification)
    if( ! _notification)
    {
        await handleNewNotification(notification)
    }
}

async function handleNewNotification(notification)
{
    // todo: send some message to me
    return await oa.upsertNotification(notification)
}

const crawler = new Crawler
(
    {
        maxConnections: 20,
        callback: handlePage,
    }
)

function homepage()
{
    const url = getPageUrl(1)
    console.log(`crawling page: ${url}`)
    crawler.queue(url);
}

homepage()
setInterval(function()
{
    homepage()
}, 3600 * 1000)