一煌往，先看結(jié)果

image.png

二，思路
1.找到頁面的所有職位糠爬，迭代后進(jìn)行請求
2.找到職位所對應(yīng)的url鏈接寇荧，迭代后，合并職位進(jìn)行請求抓取
3.實(shí)現(xiàn)手動選擇職位执隧，城市
4.實(shí)現(xiàn)一鍵入庫篩查揩抡，排選

三，上代碼

import requests
import json
import pymysql
import time
from bs4 import BeautifulSoup
import re


class Baidujob(object):
    def __init__(self):
        pass

    def mysql(self):


        conn = pymysql.Connect(host='x', user='x', password='x', database='x', port=x,
                               charset='x')
        cursor = conn.cursor()
        sql = "CREATE TABLE IF NOT EXISTS %s(ID INT(10) NOT NULL PRIMARY KEY AUTO_INCREMENT," \
              "A VARCHAR(255)," \
              "B VARCHAR(255)," \
              "C VARCHAR(255))"
        dbname = input('請輸入數(shù)據(jù)庫名:')
        cursor.execute(sql % dbname)
        print('創(chuàng)建數(shù)據(jù)庫%s成功镀琉！' % dbname)


    def get_url(self,post, page, city):
        url = 'http://zhaopin.baidu.com/api/quanzhiasync?query={}&sort_type=1&city={}&detailmode=close&rn=20&pn={}'.format(
            post, city, page)
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.3964.2 Safari/537.36'
            ,
            'Referer': 'http://zhaopin.baidu.com/quanzhi?tid=4139&ie=utf8&oe=utf8&query=python&city_sug=%E5%B9%BF%E5%B7%9E'
            , 'Host': 'zhaopin.baidu.com'
            ,
            'Cookie': 'Hm_lvt_dc173081ad0848b7d3e412373bb02119=1493714008; PSTM=1500109449; BIDUPSID=EBA58E2B59F5D325007E6FA067243233; PRY=1; BAIDUID=57829DB7D914A3AB376B0A2A7415FD8C:FG=1; Hm_lvt_da3258e243c3132f66f0f3c247b48473=1509328256; Hm_lvt_24117ca0ed302abec8cd5b93e02d18cd=1509858595; BDUSS=ltQWtHcEYxcEt6eEVkdzBUemo0R1dPZHJxdE9LOE5EbXhYaHpQMWJTVEUta0ZhQVFBQUFBJCQAAAAAAAAAAAEAAABIwC9~eWFuZ2Z1bG9uZ2hvbWUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMRtGlrEbRpaOF; MCITY=-%3A; BDRCVFR[VIIOqqdxwZ_]=mk3SLVN4HKm; PSINO=7; H_PS_PSSID=1460_21111_22075; URLTITLESALARY=%20; Hm_lvt_c676f95eebbd4fa8a59418f48090ac4d=1513303313,1514278789,1514336165,1515219239; Hm_lpvt_c676f95eebbd4fa8a59418f48090ac4d=1515219701'

        }
        # pn,0,20
        data = {
            'pn': '{}'.format(page)
            , 'rn': '20'
            , 'detailmode': 'close'
            , 'city_sug': '{}'.format(city)
            , 'sort_type': '1'
            , 'query': '{}'.format(post)

        }

        try:
            response = requests.get(url, headers=headers, params=data).text
            json_dict = json.loads(response)
            json_data = json_dict['data']
            json_main = json_data['main']
            json_datas = json_main['data']

            i = 0
            item = json_datas.get('disp_data')
            for items in item:
                ####A
                city = items.get('city')
                price = items.get('salary')
                title = items.get('title')
                buty = items.get('description_jd')
                terrace = items.get('source')
                times = items.get('lastmod')
                company_data = items.get('officialname'), ':,崗位:{},城市:{},工資:{}\n,{}平臺:{},{}\n'.format(title, city,
                                                                                                      price.split('-')[0],
                                                                                                      terrace, buty, times)
                # print(company_data)
                company = items.get('officialname')
                ###B
                phone1 = items.get('@cts')
                phone2 = items.get('@dts')
                email = items.get('email')
                companyb = items.get('officialname'), '電話:{},電話:{},EMail:{}'.format(phone1, phone2, email)

                i = i + 1
                # 變成str峦嗤，好存入MySQL
                company_str1 = ''.join(tuple(company_data))
                company_str2_phone = ''.join(tuple(companyb))
                print(i)
                print(company_str1, company_str2_phone)

                # print(company,title)
                # print(i)
                ####數(shù)據(jù)庫
            #     sql = "INSERT INTO %s(A,B)VALUES('%s','%s')"
            #     values = (dbname, company_str1, company_str2_phone)
            #     cursor.execute(sql % values)
            #     print('導(dǎo)入成功')
            # conn.commit()
            # time.sleep(0.1)
        except:
            print('response error!!!')
            return None

        else:
            pass


    def main(self):
        urls = 'http://zhaopin.baidu.com/quanzhi?tid=4139&ie=utf8&oe=utf8&query=%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90&city_sug=%E5%B9%BF%E5%B7%9E'
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.3964.2 Safari/537.36'
            ,
            'Referer': 'http://zhaopin.baidu.com/quanzhi?tid=4139&ie=utf8&oe=utf8&query=python&city_sug=%E5%B9%BF%E5%B7%9E'
            , 'Host': 'zhaopin.baidu.com'
            ,
            'Cookie': 'Hm_lvt_dc173081ad0848b7d3e412373bb02119=1493714008; PSTM=1500109449; BIDUPSID=EBA58E2B59F5D325007E6FA067243233; PRY=1; BAIDUID=57829DB7D914A3AB376B0A2A7415FD8C:FG=1; Hm_lvt_da3258e243c3132f66f0f3c247b48473=1509328256; Hm_lvt_24117ca0ed302abec8cd5b93e02d18cd=1509858595; BDUSS=ltQWtHcEYxcEt6eEVkdzBUemo0R1dPZHJxdE9LOE5EbXhYaHpQMWJTVEUta0ZhQVFBQUFBJCQAAAAAAAAAAAEAAABIwC9~eWFuZ2Z1bG9uZ2hvbWUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMRtGlrEbRpaOF; MCITY=-%3A; BDRCVFR[VIIOqqdxwZ_]=mk3SLVN4HKm; PSINO=7; H_PS_PSSID=1460_21111_22075; URLTITLESALARY=%20; Hm_lvt_c676f95eebbd4fa8a59418f48090ac4d=1513303313,1514278789,1514336165,1515219239; Hm_lpvt_c676f95eebbd4fa8a59418f48090ac4d=1515219701'

        }
        try:
            response = requests.get(urls, headers).text
            soup = BeautifulSoup(response, 'lxml')
            item = soup.find_all('div', attrs={'class': re.compile('all-jobs')})
            city = soup.find_all('div', attrs={'class': 'tabs-body'})
            for citys in city:
                cityss = citys.find_all('dd')
                for g in cityss:
                    city_name = g.get_text()
                    # city_name是城市名字
                    # print(city_name)

                    for items in item:
                        a = items.find_all('a')
                        for b in a:
                            profession = b.get_text()
                            # 職業(yè)

                            for page in range(0, 2000, 20):
                                # page=頁碼

                                self.get_url(post, city=city_name, page=page)
                                print('城市：%s,行業(yè)%s，頁碼%s' % (city_name, profession, page))
        except:
            print('main,error!!!')


if __name__ == '__main__':
    post = input('請輸入職位：')
    l = Baidujob()
    l.main()
    # post = 'Python'
    # city ='天津'
    # for page in range(0,100,20):
    #
    #     get_url(post,page,city)
    #     print(page)

    ####數(shù)據(jù)庫
    # cursor.close()
    # conn.close()

    print('CLOSE DATABASE OK!!!')