import requests
import os
from lxml import etree
def get_menu_urls(url):
    req = requests.get(url)
    req.encoding='gb2312'
    html=req.text
    selector=etree.HTML(html)
    infos=selector.xpath('//div[@class="contain"][1]/ul/li[position()<10]/a')
    url_list=[]
    for info in infos:
        a_text=info.xpath('text()')
        a_href=info.xpath('@href')
        if len(a_text)==0 or a_text[0]=='經(jīng)典影片':
            pass
        else:
            menu_url=url+a_href[0]
            print(a_text[0],menu_url)
            req2 = requests.get(menu_url)
            req2.encoding='gb2312'
            html2=req2.text
            # print(html2)
            selector2=etree.HTML(html2)
            page_total=selector2.xpath('//div[@class="co_content8"]/div[@class="x"]//text()')[1].split('/')[0].replace('共','').replace('頁(yè)','')
            print(page_total)
            list_id=selector2.xpath('//div[@class="co_content8"]/div[@class="x"]//a/@href')[0].replace('2.html','')
            print(list_id)
            for i in range(1,int(page_total)+1):
                right_url=list_id+str(i)
                # print(right_url)
                page_url=menu_url.replace('index',right_url)
                print(menu_url,right_url,page_url)
                url_list.append(page_url)

    return url_list

def get_source(url):
# print(len(url_list))
# for page_url in url_list:
    req3=requests.get(url)
    req3.encoding='gb2312'
    html3=req3.text
    selector3=etree.HTML(html3)
    infos3=selector3.xpath('//div[@class="co_content8"]//a[@class="ulink"]/@href')
    # print(len(infos3))
    for info3 in infos3:
        movie_url='http://www.ygdy8.com'+info3
    # print(page_url,moive_url)
    req4 = requests.get(movie_url)
    req4.encoding = 'gb2312'
    html4 = req4.text
    selector4 = etree.HTML(html4)
    movie_name = selector4.xpath('//div[@class="title_all"]/h1/font/text()')[0]
    movie_source=selector4.xpath('//td[@style="WORD-WRAP: break-word"]/a/@href')[0]
    print(page_url,movie_url,movie_name,movie_source)
    movie_sources=selector4.xpath('//td[@style="WORD-WRAP: break-word"]/a/@href')
    if len(movie_sources)>0:
        for movie_source in movie_sources:
            print(url, movie_url, movie_name,movie_source)
    else:
        pass

if __name__ == '__main__':
    root_url = 'http://www.ygdy8.com'
    url_list=get_menu_urls(root_url)
    for page_url in url_list:
        get_source(page_url)