安裝相關(guān)依賴(lài)包
pip3 install scrapy scrapyd scrapyd-client
尋找scrapyd服務(wù)的配置文件
sudo find / -name default_scrapyd.conf
編輯修改改文件
# 下面是編輯配置文件
[scrapyd]
eggs_dir = eggs
logs_dir = logs
items_dir =
jobs_to_keep = 5
dbs_dir = dbs
max_proc = 0
max_proc_per_cpu = 10 #可以不改 這個(gè)參數(shù)和CPU個(gè)數(shù)的乘積是爬蟲(chóng)同時(shí)運(yùn)行的最多個(gè)數(shù) 為了以后方便改成10
finished_to_keep = 100
poll_interval = 5.0
bind_address = 0.0.0.0 # 綁定的IP地址改成0.0.0.0 外網(wǎng)就可以訪問(wèn)
http_port = 6800 # 這邊是對(duì)應(yīng)的端口
debug = off
runner = scrapyd.runner
application = scrapyd.app.application
launcher = scrapyd.launcher.Launcher
webroot = scrapyd.website.Root
[services]
schedule.json = scrapyd.webservice.Schedule
cancel.json = scrapyd.webservice.Cancel
addversion.json = scrapyd.webservice.AddVersion
listprojects.json = scrapyd.webservice.ListProjects
listversions.json = scrapyd.webservice.ListVersions
listspiders.json = scrapyd.webservice.ListSpiders
delproject.json = scrapyd.webservice.DeleteProject
delversion.json = scrapyd.webservice.DeleteVersion
listjobs.json = scrapyd.webservice.ListJobs
daemonstatus.json = scrapyd.webservice.DaemonStatus
修改scrapy的scrapy.cfg文件配置
[settings]
default = Spideraddress.settings
[deploy: address]#address為用于啟動(dòng)的名稱(chēng)
url = http://localhost:6800/#scrapyd服務(wù)地址
project = Spideraddress#項(xiàng)目名稱(chēng)
啟動(dòng)測(cè)試
scrapy list#沒(méi)報(bào)錯(cuò)即scrapy安裝成功
scrapyd #輸出正常即scrapyd啟動(dòng)成功
訪問(wèn)scrapyd服務(wù)web端
http://127.0.0.1:6800
最后編輯于 ：
?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請(qǐng)聯(lián)系作者
