from lxml import etree
import pandas as pd
import requests
import pickle
import sys
import os


def get_page_content(url):
    """鏈接網(wǎng)頁谁尸，獲取頁面內(nèi)容"""
    head = 'http://ttc.seu.edu.cn'
    header = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0',
              'referer': 'http://ttc.seu.edu.cn'}

    while True:
        try:
            print(f'\033[1;32m正在下載: {url}\033[0m')
            content = requests.get(url, headers=header, timeout=10)
        except KeyboardInterrupt:
            print('停止獲取網(wǎng)頁成果错忱，即將退出...')
            sys.exit()
        except:
            print('\033[1;31m網(wǎng)頁內(nèi)容獲取失敗霍弹，重新鏈接...\033[0m')
            continue
        break
    html = etree.HTML(content.text)
    title = html.xpath('//div[@class="article-title-all"]/text()')[0]  # 成果標(biāo)題
    # data1 = html.xpath('//dl/*/text()')   # 描述屬性的標(biāo)題與內(nèi)容,內(nèi)容為空的部分會缺失谒撼，沒有占位
    technical_filed = html.xpath('//dl/dd/span/text()')  # 技術(shù)領(lǐng)域
    technical_filed = ', '.join(technical_filed)
    technical_describe = html.xpath('//div[@class="gray-box-no-border resource-big-detail"]/p/text()')  # 成果介紹
    technical_describe = '<br>'.join(technical_describe)
    content1 = html.xpath('//div[@class="col-sm-6 col-xs-12"]/dl/dd')  # 證書楼咳，單位熄捍，時間，階段母怜，領(lǐng)域余耽，產(chǎn)權(quán)，歸屬
    content2 = html.xpath('//div[@class="col-md-6 col-xs-12"]/div/dl/dd')  # 合作苹熏，單位介紹
    pics = html.xpath('//div[@class="sp-wrap"]/a')  # 獲取圖片節(jié)點(diǎn)
    pics_link = [head + v.values()[0] for v in [a.attrib for a in pics]]  # 獲取標(biāo)簽屬性里面圖片鏈接,組成列表
    pics_links = [', '.join(pics_link)]
    contact = html.xpath('//div[@class="gray-box-no-border"]/p/text()')  # 聯(lián)系方式
    content1 = [i.text for i in content1]
    content2 = [j.text for j in content2]
    content1[4] = technical_filed

    content = [title] + content1 + [technical_describe] + content2 + contact + pics_links + [url]
    save_to_excel(content, title)  # 保存獲取的內(nèi)容


def save_to_excel(content,title):
    """把內(nèi)容保存到excel表格"""
    print('保存數(shù)據(jù)...')
    file_name = 'ddcg.xlsx'
    keylist = ['成果名稱','知識產(chǎn)權(quán)證書號','單位名稱','發(fā)布時間','項(xiàng)目階段','技術(shù)領(lǐng)域','知識產(chǎn)權(quán)情況','成果歸屬','成果介紹',
               '項(xiàng)目負(fù)責(zé)人', '合作方式','轉(zhuǎn)讓價格','法人代表','單位規(guī)模','單位性質(zhì)','電話','郵箱','地址','圖片鏈接','原始網(wǎng)頁鏈接']
    tem_zip = list(zip(keylist,content))

    c = {}  # 保存結(jié)果
    for key, value in tem_zip:
        c[key] = [value]
    pd_c = pd.DataFrame(c)  # 轉(zhuǎn)化為表格數(shù)據(jù)
    if os.path.exists(file_name):
        data = pd.read_excel(file_name)
        data = pd.concat([data, pd_c])
        data.to_excel(file_name, index=False)
        print('%s: 追加完畢碟贾！' % title)
    else:
        pd_c.to_excel(file_name, index=False)
        print('%s: 保存完畢！' %title)


def get_page_link(list_link):
    """獲取頁面成果鏈接以及下一頁列表鏈接"""
    head = 'http://ttc.seu.edu.cn'
    header = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0',
              'referer': 'http://ttc.seu.edu.cn'}
    num = list_link.split('=')[-1]
    print(f'正在獲取第 {num} 頁成果列表...')
    while True:
        try:
            page_data  = requests.get(list_link,headers=header,timeout=10)
            break
        except KeyboardInterrupt:
            print('主動退出下載：\n已經(jīng)下載到：%s' % list_link)
            with open('list_link.txt','w') as lobj:
                lobj.write(list_link)
        except:
            print('獲取網(wǎng)頁錯誤轨域，重新下載...')
            continue

    data_xpath = etree.HTML(page_data.text)
    cc_list_link = data_xpath.xpath('//a[@class="article-title"]')  # 獲取成果鏈接標(biāo)簽元素
    alist = [head + link[link.keys()[0]] for link in [d.attrib for d in cc_list_link]]  # 得到本頁面成果列表鏈接
    # print(alist)
    next_list_e = data_xpath.xpath('//nav/ul/li[last()]/a')  # 獲取下一頁鏈接標(biāo)簽元素
    next_list_page_e = [v['href'] for v in [a.attrib for a in next_list_e]][0]  # 獲取標(biāo)簽的下一頁鏈接內(nèi)容
    # print(next_list_page_e)
    get_next_list = head + next_list_page_e if next_list_page_e.startswith('/abutment') else ''

    return get_next_list, alist


if __name__ == '__main__':
    url = 'http://ttc.seu.edu.cn/abutment/technologyachievementsearch?pageIndex=2'
    while True:
        next_list_link, cclist = get_page_link(url)
        while cclist:
            link = cclist.pop()
            while True:
                try:
                    get_page_content(link)
                    with open('download_link.txt','a') as dobj:
                        dobj.write(link + '\n')  # 保存已下載鏈接
                    break
                except KeyboardInterrupt:
                    print('主動退出袱耽，保存進(jìn)度...')
                    with open('cc_list.txt','w') as cobj:
                        cclist.append(link)
                        # pickle.dump(cclist,cobj)
                        cobj.write(cclist)
                    with open('next_list.txt','w') as lobj:
                        lobj.write(next_list_link)
                    exit()
                except Exception as e :
                    print('其他錯誤。干发。朱巨。', e)
                    # continue
        if next_list_link == '':
            print('下載完畢，退出下載...')
            for i in ['cc_list.txt','next_list.txt']:
                if os.path.exists(i):
                    os.remove(i)
            sys.exit()
        url = next_list_link