import requests as re
import json
import pandas as pd
import time
import random
'''其中三個編碼是全國阔逼、深圳和廣州，但是哪個是哪個不記得了'''
#%E6%B7%B1%E5
#%E5%B9%BF%E5%B7%9E
#%E5%85%A8%E5%9B%BD
#獲取網頁url
def get_url(keyword,pn):
   base_url1 = "https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90?labelWords=&fromSearch=true&suginput="    #不需要cookie的url
   base_url = "https://www.lagou.com/jobs/positionAjax.json?px=default&city=%E6%B7%B1%E5%BD&needAddtionalResult=false"

   base_headers = {"Accept": "application/json, text/javascript, */*; q=0.01",
                   "Host": "www.lagou.com",
                   "Referer": "https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90?labelWords=&fromSearch=true&suginput=",
                   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
                   }
   data = {
           "first": "true",
           "pn": "{}".format(pn),
           "kd": "{}".format(keyword)
           }

   session = re.Session()      #創(chuàng)建cookie存儲
   session.get(url=base_url1,headers=base_headers)        #通過網址url1建立cookie
   response = session.post(url=base_url, headers=base_headers, data=data)
   response.encoding = 'utf-8'
   #print(response)

   return response

#解析網頁信息
def total_Count(response):

   page = response.json()
   # print(page)
   total_count = page['content']['positionResult']['totalCount']   #totalCount為總個數
   pn_count = int(total_count)//15 + 1
   #頁數
   print('職位總數{},共{}頁'.format(total_count,pn_count))
   return pn_count

def parse_url(response):
   page = response.json()
   for i in range(1,16):
       jobs_list = page['content']['positionResult']['result']
       print(jobs_list)
       page_info_list = []             #用于存儲data
       for i in jobs_list:
           job_info = []
           job_info.append(i['city'])
           job_info.append(i['companyFullName'])
           job_info.append(i['companyShortName'])
           job_info.append(i['companySize'])
           job_info.append(i['financeStage'])
           job_info.append(i['district'])
           job_info.append(i['positionName'])
           job_info.append(i['workYear'])
           job_info.append(i['education'])
           job_info.append(i['salary'])
           job_info.append(i['positionAdvantage'])
           job_info.append(i['industryField'])
           page_info_list.append(job_info)
       print(page_info_list)
       return page_info_list

def save_data(page_info_list):
   columns = ['城市','公司全名', '公司簡稱', '公司規(guī)模', '融資階段', '區(qū)域', '職位名稱', '工作經驗', '學歷要求', '工資', '職位福利', '行業(yè)']
   df = pd.DataFrame(data=page_info_list,
                     columns=columns)
   #print(df)
   df.to_csv('/Users/liyili2/Downloads/data2.csv', index=False,encoding='utf-8')
   print('保存完成')

#     #調用中心
def main():
   keyword = input('輸入查找內容:')      #輸入搜索內容
   a = get_url(keyword,pn=1)            #獲取respones
   b = total_Count(a)
   print(b)# 獲得數據總個數和頁數
   total_info = []                     #用來儲存每頁parse_url
   for i in range(1,int(b)+1):         #實現翻頁效果
       a = get_url(keyword, pn=i)
       c = parse_url(a)
       total_info += c
       time.sleep(20)
       print('成功獲取第{}頁'.format(i))
   d = save_data(total_info)

if __name__ =="__main__":
   main()