輿情監(jiān)測是對互聯(lián)網(wǎng)上公眾的言論和觀點進行監(jiān)視和預測的行為.監(jiān)測技術(shù)大多是基于爬蟲的, 如果我們把相關熱點事件的關鍵詞, 用搜索引擎進行搜索, 并將結(jié)果保存到本地,就實現(xiàn)了輿情監(jiān)測的第一環(huán)節(jié):實時獲取互聯(lián)網(wǎng)數(shù)據(jù)

輿情監(jiān)測.png

初步實現(xiàn)效果

獲取數(shù)據(jù).gif

實現(xiàn)代碼

import requests
from lxml import etree
import os
import sys

def getData(wd):
    # 設置用戶代理頭
    headers = {
        # 設置用戶代理頭(為狼披上羊皮)
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
    }
    # 構(gòu)造目標網(wǎng)址
    target_url = "https://www.baidu.com/s?wd="+str(wd)
    # 獲取響應
    data = requests.get(target_url, headers = headers)
    # xpath格式化
    data_etree = etree.HTML(data.content)
    # 提取數(shù)據(jù)列表
    content_list = data_etree.xpath('//div[@id="content_left"]/div[contains(@class, "result c-container")]')
    # 定義返回的字符串
    result = ""
    # 獲取標題, 內(nèi)容, 鏈接
    for content in content_list:
        result_title = "<標題>  "
        bd_title = content.xpath('.//h3/a')
        for bd_t in bd_title:
            result_title += bd_t.xpath('string(.)')

        result_content = "<內(nèi)容>  "
        bd_content = content.xpath('.//div[@class="c-abstract"]')
        for bd_c in bd_content:
            result_content += bd_c.xpath('string(.)')

        result_link = "<鏈接>  "+str(list(content.xpath('.//div[@class="f13"]/a[@class="c-showurl"]/@href'))[0])


        result_list = [result_title, "\n" , result_content , "\n", result_link, "\n", "\n"]
        for result_l in result_list:
            result += str(result_l)
    return result


# 保存為文件

def saveDataToFile(file_name, data):
    # 建立文件夾
    if os.path.exists("./data/"):
        pass
    else:
        os.makedirs("./data/")

    with open("./data/"+file_name+".txt", "w+") as f:
        f.write(data)

def main():
    wd = ""
    try:
        wd = sys.argv[1]
    except:
        pass
    if (len(wd) == 0):
        wd = "火影"
    str_data = getData(wd)
    print(str_data)
    saveDataToFile(wd, str_data)

if __name__ == '__main__':
    main()