簡(jiǎn)單的fetch并行改串行控制
/**
* 基于fetch的并行控制
* 【鄭重聲明】 該代碼著作權(quán)歸瞌睡蟲子所有纺念。其他第三方使用該代碼用于商用活動(dòng)，引起的法律糾紛與作者無關(guān)，同時(shí)作者保留追責(zé)權(quán)力曾沈。
*
* 2024-12-29
* 后臺(tái)这嚣，序列http下載，后臺(tái)爬蟲框架http部分
* ____________________________________________________________________________________________________________________________________
*/
// 隊(duì)列抓取限制抓取并發(fā)
class FetchQueue {
constructor(maxConcurrentRequests, interval = 1000) {
console.log("創(chuàng)建爬蟲");
this.maxConcurrentRequests = maxConcurrentRequests;
this.currentlyRunning = 0;
this.queue = [];
this.interval = interval;
}
add(url) {
return new Promise((resolve, reject) => {
const task = () => {
setTimeout(() => {
if (this.currentlyRunning < this.maxConcurrentRequests) {
this.currentlyRunning++;
// 這里添加異步隊(duì)列批量處理方法
fetch(url)
.then(response => response.text())
.then(data => {
this.currentlyRunning--;
resolve(data);
this.next();
})
.catch(error => {
this.currentlyRunning--;
reject(error);
this.next();
});
} else {
this.queue.push(task);
}
}, this.interval);
};
if (this.currentlyRunning < this.maxConcurrentRequests) {
task();
} else {
this.queue.push(task);
}
});
}
next() {
if (this.queue.length > 0 && this.currentlyRunning < this.maxConcurrentRequests) {
const task = this.queue.shift();
task();
}
}
}
接受content.js的消息塞俱，控制后臺(tái)下載姐帚，返回給content.js渲染都界面
backgroud.js
// 使用 FetchQueue，并設(shè)置不同的間隔時(shí)間 1 障涯， 600
const maxConcurrentRequests = 1; // 設(shè)置最大并發(fā)請(qǐng)求數(shù)
const fetchQueue = new FetchQueue(maxConcurrentRequests, 620);
chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
if (request.method === "fetchUrl") {
// 將 fetch 請(qǐng)求添加到隊(duì)列
fetchQueue.add(request.url)
.then(function (html) {
// 使用正則表達(dá)式抽取人氣值
var regex = /(\d+)\s*人次/;
var match = html.match(regex);
if (match && match[1]) {
// match[1] 就是人氣值
var popularity = match[1];
sendResponse({
id: request.id,
msg: popularity,
flag: true
});
// console.log('人氣值:', popularity);
} else {
sendResponse({
id: request.id,
msg: "未找到人氣值",
flag: false
});
console.log('未找到人氣值');
}
})
.catch(error => sendResponse({
id: request.id,
msg: "請(qǐng)求失敗",
flag: false
}));
return true; // 表示響應(yīng)是異步的
} else if (request.method === "clear") {
fetchQueue.queue = [];
sendResponse({
msg: "隊(duì)列清空成功",
flag: true
})
return false; // 不處理
} else if (message.method === "openPopup") {
chrome.action.openPopup();
}
return false; // 不處理
});
// 激活標(biāo)簽頁(yè)
chrome.tabs.onActivated.addListener(function (activeInfo) {
// 你可以獲取激活的標(biāo)簽頁(yè)的詳細(xì)信息
chrome.tabs.get(activeInfo.tabId, function (tab) {
// 在這里你可以根據(jù)激活的標(biāo)簽頁(yè)執(zhí)行一些操作
if (tab.url.match(/目標(biāo)網(wǎng)頁(yè)/)) {
console.log('Active tab URL: ' + tab.url);
chrome.tabs.sendMessage(activeInfo.tabId, {
method: "clear"
});
}
});
});
與backgroud.js通訊罐旗，發(fā)送任務(wù)消息等待結(jié)果，再渲染
contet.js
/**
* 瞌睡蟲子
*
* 2024-11-5
* 人氣值抓去顯示
*/
console.log('開始工作啦：', location.href);
// 隨機(jī)唯一ID
function generateUUID() {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
var r = Math.random() * 16 | 0,
v = c === 'x' ? r : (r & 0x3 | 0x8);
return v.toString(16);
});
}
function getRQ() {
var res = {}
document.querySelectorAll("li em a:not([class='fetch'])").forEach(function (dd) {
// 獲取當(dāng)前元素的 'href' 屬性
var url = dd.href;
var uuid = "L_" + generateUUID();
res[uuid] = dd;
// console.log(url);
chrome.runtime.sendMessage({
method: "fetchUrl",
url: url,
id: uuid
}, function (response) {
var id = response.id;
var msg = response["msg"];
if (msg && res[id] && response.flag) {
var node = res[id];
node.className += "fetch";
node.parentElement.nextElementSibling.innerHTML += ' <span style="font-weight: normal;font-size: 12px;color: black;">人氣:' + msg + "</span>";
res[id] = undefined;
}
});
});
}
var MutationObserver = window.MutationObserver || window.WebKitMutationObserver || window.MozMutationObserver;
const options = {
childList: true
// attributes: true,
// characterData: true,
// subtree: true,
// attributeOldValue: true,
// characterDataOldValue: true
};
// 創(chuàng)建MutationObserver實(shí)例唯蝶，返回一個(gè)觀察者對(duì)象
const mutation = new MutationObserver(function (mutationRecoards, observer) {
for (const recoard of mutationRecoards) {
if (recoard.type === 'childList') {
// getRQ(recoard.addedNodes);
// setTimeout(getRQ, 500);
console.log("加載新數(shù)據(jù)…");
setTimeout(function () {
var res = {};
recoard.addedNodes.forEach(function (dd) {
if (dd.nodeType == 1 && dd.nodeName == "LI") {
// 獲取當(dāng)前元素的 'href' 屬性
var aa = dd.querySelector("em a");
var url = aa.href;
var uuid = "L_" + generateUUID();
res[uuid] = aa;
// console.log(url);
chrome.runtime.sendMessage({
method: "fetchUrl",
url: url,
id: uuid
}, function (response) {
var id = response.id;
var msg = response["msg"];
if (msg && res[id] && response.flag) {
var node = res[id];
node.className += "fetch";
node.parentElement.nextElementSibling.innerHTML += ' <span style="font-weight: normal;font-size: 12px;color: black;">人氣:' + msg + "</span>";
res[id] = undefined;
}
});
};
});
}, 500);
}
}
});
// 當(dāng)前頁(yè)面激活
// content.js
chrome.runtime.onMessage.addListener(function (message, sender, sendResponse) {
if (message && message.method === "clear") {
// 標(biāo)簽頁(yè)被激活九秀，執(zhí)行相關(guān)操作
console.log("頁(yè)面激活，繼續(xù)抓取…");
try {
chrome.runtime.sendMessage({
method: "clear"
},function(response) {
getRQ();
});
} catch (error) {
}
}
return false;
});
// 對(duì)觀察者添加需要觀察的元素粘我，并設(shè)置需要觀察元素的哪些方面
document.querySelectorAll('ul').forEach(function (targetElement) {
mutation.observe(targetElement, options);
});
// 清空爬取隊(duì)列
chrome.runtime.sendMessage({
method: "clear"
});
// 發(fā)消息請(qǐng)求
getRQ();
