以爬取NASDAQ的股票數(shù)據(jù)為例
依賴
from lxml import html //獲取網(wǎng)頁信息
import requests //地址請求數(shù)據(jù)
from time import sleep //延時用
import json //生成json
import argparse //python 執(zhí)行參數(shù)讀取 暫時不用
from random import randint //生成延時隨機(jī)數(shù)
import pymongo //寫入mongodb
獲取數(shù)據(jù)
設(shè)置請求頭
headers = {
"Referer": "http://www.nasdaq.com",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36"
}
請求網(wǎng)頁數(shù)據(jù)
url = "http://www.nasdaq.com/symbol/%s" % (ticker)
response = requests.get(url, verify=False)
print("respose code:%d" % (response.status_code))
if response.status_code != 200:
raise ValueError("Invalid Response Received From Webserver")
解析和組裝數(shù)據(jù)（解析規(guī)則參考html.fromstring.xpath）
parser = html.fromstring(response.text)
xpath_head = "http://div[@id='qwidget_pageheader']//h1//text()"
xpath_key_stock_table = '//div[@class="row overview-results relativeP"]//div[contains(@class,"table-table")]/div'
xpath_open_price = '//b[contains(text(),"Open Price:")]/following-sibling::span/text()'
xpath_open_date = '//b[contains(text(),"Open Date:")]/following-sibling::span/text()'
xpath_close_price = '//b[contains(text(),"Close Price:")]/following-sibling::span/text()'
xpath_close_date = '//b[contains(text(),"Close Date:")]/following-sibling::span/text()'
xpath_key = './/div[@class="table-cell"]/b/text()'
xpath_value = './/div[@class="table-cell"]/text()'
raw_name = parser.xpath(xpath_head)
key_stock_table = parser.xpath(xpath_key_stock_table)
raw_open_price = parser.xpath(xpath_open_price)
raw_open_date = parser.xpath(xpath_open_date)
raw_close_price = parser.xpath(xpath_close_price)
raw_close_date = parser.xpath(xpath_close_date)
company_name = raw_name[0].replace("Common Stock Quote & Summary Data", "").strip() if raw_name else ''
open_price = raw_open_price[0].strip() if raw_open_price else None
open_date = raw_open_date[0].strip() if raw_open_date else None
close_price = raw_close_price[0].strip() if raw_close_price else None
close_date = raw_close_date[0].strip() if raw_close_date else None
# Grabbing ans cleaning keystock data
for i in key_stock_table:
key = i.xpath(xpath_key)
value = i.xpath(xpath_value)
key = ''.join(key).strip().replace(".", "_")
value = ' '.join(''.join(value).split())
key_stock_dict[key] = value
nasdaq_data = {
"company_name": company_name,
"ticker": ticker,
"url": url,
"open_price": open_price,
"open_date": open_date,
"close_price": close_price,
"close_date": close_date,
"key_stock_data": key_stock_dict
}
return nasdaq_data
主函數(shù)執(zhí)行
symbols = ['aapl', 'gluu', 'mu', 'ntap', 'msft', 'intc', 'znga', 'csco', 'siri', 'jd', 'fb', 'nvda', 'bl', 'ftnt', 'chrs', 'loco', 'catm', 'cnce', 'fizz', 'acor', 'fldm', 'sptn', 'cent', 'xent', 'adap', 'gpro', 'brks', 'sgms', 'iova', 'aaon', 'eigi', 'amzn', 'nflx', 'tsla']
for ticker in symbols:
print("Fetching data for %s" % (ticker))
scraped_data = parse_finance_page(ticker)
print("Writing scraped data to output file")
with open('%s-summary.json' % (ticker), 'w') as fp:
json.dump(scraped_data, fp, indent=4, ensure_ascii=False)
myclient = pymongo.MongoClient('mongodb://localhost:27107/')
mydb = myclient['resthub']
mycol = mydb["stocks"]
x = mycol.insert_one(scraped_data)
print(x.inserted_id)
結(jié)果
Fetching data for aapl
respose code:200
Parsing http://www.nasdaq.com/symbol/aapl
Writing scraped data to output file
5d44642d79c3f9765ab465fa
Fetching data for gluu
respose code:200
Parsing http://www.nasdaq.com/symbol/gluu
Writing scraped data to output file
5d44642f79c3f9765ab465fc
Fetching data for mu
respose code:200
Parsing http://www.nasdaq.com/symbol/mu
Writing scraped data to output file
5d44643179c3f9765ab465fe
Fetching data for ntap
respose code:200
Parsing http://www.nasdaq.com/symbol/ntap
Writing scraped data to output file
5d44643579c3f9765ab46600
需要mongodb已啟動在端口27107
