# get_page() -> parse_page() -> filter_job() -> send()
'''
raw_html = []
for i in range(1, 30):
    page = get_page()
    raw_html.append(page)

all_jobs = []
for html in raw_html:
    jobs = parse(html)
    all_jobs.append(jobs)

for job in all_jobs:
    result = filter_job(job)
    if result:
        send(job)
'''

# Spider -> Parser -> Job

'''
s = Spider()
raw_pages = s.crawl(url)
p = Parser(raw_pages)
jobs = p.get_jobs()

for j in jobs:
    if j.is_today():
        j.send_to_me()
'''
from selenium.webdriver import Chrome
from bs4 import BeautifulSoup
import time
# https://www.lagou.com/zhaopin/qukuailian/12/
class Spider:
    def __init__(self, index_url, page_range):
        self.page_range = page_range + 1
        self.index_url = index_url
        self.raw_pages = []
        self.boot()

    def boot(self):
        self.chrome = Chrome(executable_path='./chromedriver')
        self.chrome.start_client()

    def crawl(self):
        for num in range(1, self.page_range):
            full_url = f'{self.index_url}{num}/'
            self.chrome.get(full_url)
            print('Wait for loading page')
            time.sleep(3)
            single_html = self.chrome.page_source
            #本來這個(gè)瀏覽器應(yīng)該打開就處理晓殊，但是也可以一次性網(wǎng)頁都加載完再挨個(gè)處理断凶，這個(gè)可以存內(nèi)容的。巫俺，
            self.raw_pages.append(single_html)
            print('Done')


class Parser:

    def __init__(self, raw_pages):
        self.raw_pages = raw_pages
        self.jobs = []

        self.parse()

    def parse(self):
        for html in self.raw_pages:
            soup = BeautifulSoup(html, 'html.parser') #這里注意是html.parser
            time_sel = 'ul span.format-time'
            comp_sel = 'ul .company_name > a'
            link_sel = 'ul a.position_link'

            time_els = soup.select(time_sel) # list
            comp_els= soup.select(comp_sel)  # list
            link_els= soup.select(link_sel)  # list
            for t,c,l in zip(time_els, comp_els, link_els):

                cell = {  #字典方便查詢
                    'time':t.text,
                    'comp':c.text,
                    'link':l.get('href')
                }
                self.jobs.append(cell)
                # [{},{}]
    def get_jobs(self):
        return [Job(j) for j in self.jobs]



class Job:

    def __init__(self,data):

        self.time = data.get('time')
        self.comp = data.get('comp')
        self.link = data.get('link')


    def is_today(self):
        return ':' in self.time # -> T or F

    def send(self):
        pass

    def save_into_csv(self):
        pass


s = Spider(
    index_url='https://www.lagou.com/zhaopin/qukuailian/',
    page_range=2
)
s.crawl()
p = Parser(s.raw_pages)
jobs = p.get_jobs()
for j in jobs:
    if j.is_today():
        print(j.comp,j.link)