fiddler插件開發(fā)-DouYinCapture
文集 移動端網(wǎng)頁端爬蟲
源碼
-
前言
前面幾篇文章介紹了app的自動化柒巫，但是光這樣只是去掉人工操作奉件，app運行時的請求并沒有抓下來踪宠。下面介紹如何通過代理實現(xiàn)數(shù)據(jù)的抓取 — DouYinCapture。
-
介紹
DouYinCapture是fiddler的最基礎(chǔ)最簡單的一個插件即實現(xiàn)了IFiddlerExtension, IAutoTamper, IAutoTamper2接口的dll類庫，只需要將這個dll文件加入到fiddler/scripts/下即能加載插件并按照我們的意愿過濾請求
部分代碼
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Fiddler;
using RabbitMQ.Client;
namespace DouYinCaptuer
{
public class Douyin : IFiddlerExtension, IAutoTamper, IAutoTamper2
{
private static IModel channel;
private void sendTask(byte[] bs, string key)
{
try
{
channel.BasicPublish(exchange: "DouYin",
routingKey: key,
basicProperties: null,
body: bs);
FiddlerApplication.Log.LogString("rabbitmq: send successfully");
}
catch (Exception)
{
FiddlerApplication.Log.LogString("rabbitmq: send timeout");
}
}
//private static FileStream fs = new FileStream("I:\\fiddlerlog\\response\\douyin.txt", FileMode.Append, FileAccess.Write);
public void AutoTamperRequestAfter(Session oSession)
{
}
public void AutoTamperRequestBefore(Session oSession)
{
}
public void AutoTamperResponseAfter(Session oSession)
{
if (oSession.responseCode == 200)
{
String path = oSession.PathAndQuery.Split('?')[0];
String queue = "";
String key = "";
switch (path)
{
case "/aweme/v1/feed/":
// 首頁列表
queue = "douyin.author";
key = "douyin.author.feed";
break;
case "/aweme/v1/user/":
// 主播信息
queue = "douyin.author";
key = "douyin.author.info";
break;
case "/aweme/v1/aweme/post/":
// 主播作品
queue = "douyin.author";
key = "douyin.author.post";
break;
case "/aweme/v1/aweme/favorite/":
// 主播喜歡的視頻
queue = "douyin.author";
key = "douyin.author.favorite";
break;
case "/aweme/v1/user/following/list/":
// 主播關(guān)注
queue = "douyin.author";
key = "douyin.author.following";
break;
case "/aweme/v1/user/follower/list/":
// 主播粉絲
queue = "douyin.author";
key = "douyin.author.follower";
break;
case "/aweme/v1/music/detail/":
// 音樂人信息
queue = "douyin.music";
key = "douyin.music.detail";
break;
case "/aweme/v1/music/fresh/aweme/":
// 使用某音樂最新的作品
queue = "douyin.music";
key = "douyin.music.fresh";
break;
case "/aweme/v1/music/aweme/":
// 使用某音樂的作品中最熱的
queue = "douyin.music";
key = "douyin.music.hot";
break;
case "/aweme/v1/comment/list/":
//評論列表
queue = "douyin.author";
key = "douyin.author.comment";
break;
case "/rest/n/feed/hot":
queue = "kuaishou.author";
key = "kuaishou.author.kol";
break;
default:
return;
}
if (queue != "")
{
// # 提取uid 放到response中
String uid_pattern = "user_id=(?<uid>\\d{1,13})&";
String mid_pattern = "music_id=(?<mid>\\d{1,20})&";
Match uid_match = Regex.Match(oSession.PathAndQuery, uid_pattern);
Match mid_match = Regex.Match(oSession.PathAndQuery, mid_pattern);
/* String pattern = "\"uid\" ?: ?\"(?<uid>\\d{11,12})\"";
MatchCollection matches = Regex.Matches(oSession.GetResponseBodyAsString(), pattern);
foreach (Match match in matches)
{
fs.Write(Encoding.UTF8.GetBytes(match.Groups["uid"]+"\r\n"), 0, match.Groups["uid"].Length+"\r\n".Length);
}
*/
string response = oSession.GetResponseBodyAsString().Trim();
string post_time = ", \"post_time\":\"" + GetTimestamp().ToString() + "\"}";
response = response.Substring(0, response.Length - 1) + post_time;
if (uid_match.Success)
{
string uid = ", \"uid\":\"" + uid_match.Groups["uid"] + "\"}";
response = response.Substring(0, response.Length - 1) + uid;
}
if (mid_match.Success)
{
string mid = ", \"mid\":\"" + mid_match.Groups["mid"] + "\"}";
response = response.Substring(0, response.Length - 1) + mid;
}
sendTask(Encoding.UTF8.GetBytes(response), key);
}
}
}
public void AutoTamperResponseBefore(Session oSession)
{
}
public void OnBeforeReturningError(Session oSession)
{
}
public void OnBeforeUnload()
{
FiddlerApplication.Log.LogString("卸載抖音插件成功");
// fs.Close();
channel.Close();
channel.Dispose();
}
public void OnLoad()
{
FiddlerApplication.Log.LogString("加載抖音插件成功");
try
{
ConnectionFactory rbmqfactory = new ConnectionFactory()
{
HostName = "118.190.112.223",
Port = 5672,
UserName = "admin",
Password = "avGX9IHGeUVVcs5B"
};
IConnection con = rbmqfactory.CreateConnection();
channel = con.CreateModel();
FiddlerApplication.Log.LogString("連接RabbitMQ成功");
}
catch (Exception)
{
FiddlerApplication.Log.LogString("rabbitmq: send timeout");
}
}
public void OnPeekAtResponseHeaders(Session oSession)
{
}
public static long GetTimestamp()
{
TimeSpan ts = DateTime.Now.ToUniversalTime() - new DateTime(1970, 1, 1);//ToUniversalTime()轉(zhuǎn)換為標(biāo)準(zhǔn)時區(qū)的時間,去掉的話直接就用北京時間
return (long)ts.TotalSeconds;//獲取10位
}
}
}
代碼很簡單茫藏，唯一需要注意的用了AssemblyInfo.cs的配置必須指定fiddler的最低版本民鼓，如果不指定薇芝，到時候fiddler既不報錯也不能生效插件就很騷了。下面是我的fiddler的版本
> [assembly: Fiddler.RequiredVersion("5.0.20182.28034")]
FIDDLER4.png
- 至于如何將手機(jī)代理設(shè)置到fiddler丰嘉，相關(guān)教程網(wǎng)上一大堆
