??承接上一篇【渲染逆向】Hook D3D API.
DX11的Hook流程
??程序的入口點(DllMain)在win32_libentry.cpp中
BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
{
if(ul_reason_for_call == DLL_PROCESS_ATTACH)
{
BOOL ret = add_hooks();
SetLastError(0);
return ret;
}
return TRUE;
}
??在函數(shù)add_hooks的結尾會調(diào)用LibraryHooks::RegisterHooks()
void LibraryHooks::RegisterHooks()
{
BeginHookRegistration();
for(LibraryHook *lib : LibList())
lib->RegisterHooks();
EndHookRegistration();
}
??LibList函數(shù)返回LibraryHook對象的列表或油，而向列表中加入對象贞间，是LibraryHook調(diào)用構造函數(shù)的時候。
LibraryHook::LibraryHook()
{
LibList().push_back(this);
}
??顯然LibraryHook是一個基類韭脊，會有超級多需要Hook的庫來繼承這一類，而我們現(xiàn)在關心的是DX11的庫信姓。
class D3D11Hook : LibraryHook
{
public:
void RegisterHooks();
private:
static D3D11Hook d3d11hooks;
//something....
}
??可以發(fā)現(xiàn)D3D11Hooks有一個靜態(tài)成員适篙，當這個類型初始化時，會自動生成一個D3D11Hook對象晾腔，并調(diào)用父類LibraryHook的構造函數(shù)，將自身加入Lib列表中啊犬。
??當函數(shù)add_hooks遍歷列表灼擂，調(diào)用RegisterHooks時，會調(diào)用D3D11Hook::RegisterHooks椒惨。
void RegisterHooks()
{
//something....
CreateDevice.Register("d3d11.dll", "D3D11CreateDevice", D3D11CreateDevice_hook);
CreateDeviceAndSwapChain.Register("d3d11.dll", "D3D11CreateDeviceAndSwapChain",
D3D11CreateDeviceAndSwapChain_hook);
}
??這里會調(diào)用HookedFunction::Register缤至，然后調(diào)用LibraryHooks::RegisterFunctionHook，將Hook所需的數(shù)據(jù)（dll庫名稱康谆、函數(shù)名稱领斥、原始函數(shù)地址、hook函數(shù)地址）記錄到數(shù)據(jù)結構中沃暗，之后統(tǒng)一用IAT hook進程函數(shù)hook月洛，如果有需要，可以在這里修改為其他hook類型孽锥，例如：
void LibraryHooks::RegisterFunctionHook(const char *libraryName, const FunctionHook &hook)
{
//something...
if(!_stricmp(libraryName, "d3d11.dll") || !_stricmp(libraryName, "dxgi.dll"))
{
std::cout << "inline hook " << libraryName
<< "\t-\t" << hook.function.c_str() << std::endl;
*hook.orig = GetProcAddress(GetModuleHandleA(libraryName), hook.function.c_str());
DetourTransactionBegin();
DetourUpdateThread(GetCurrentThread());
DetourAttach((PVOID *)hook.orig, hook.hook);
DetourTransactionCommit();
}
else
{
s_HookData->DllHooks[strlower(rdcstr(libraryName))].FunctionHooks.push_back(hook);
}
}
??這里我就修改了一下嚼黔，假如是d3d11.dll或dxgi.dll的函數(shù)，則進行inline hook惜辑，而其他庫函數(shù)則保持原樣唬涧。
??dxgi.dll的Hook同理:
class DXGIHook : LibraryHook
{
public:
void RegisterHooks()
{
LibraryHooks::RegisterLibraryHook("dxgi.dll", NULL);
CreateDXGIFactory.Register("dxgi.dll", "CreateDXGIFactory", CreateDXGIFactory_hook);
CreateDXGIFactory1.Register("dxgi.dll", "CreateDXGIFactory1", CreateDXGIFactory1_hook);
CreateDXGIFactory2.Register("dxgi.dll", "CreateDXGIFactory2", CreateDXGIFactory2_hook);
GetDebugInterface.Register("dxgi.dll", "DXGIGetDebugInterface", DXGIGetDebugInterface_hook);
GetDebugInterface1.Register("dxgi.dll", "DXGIGetDebugInterface1", DXGIGetDebugInterface1_hook);
}
??這些都是Hook需要查詢地址的函數(shù)，而類似SetRenderTargets之類的虛表函數(shù)的Hook方式盛撑，放到下一節(jié)來找碎节。
DX11截幀流程
??RenderDoc按鍵或調(diào)用提供的api截幀都是調(diào)用core.h中的RenderDoc::TriggerCapture(uint32_t numFrames)函數(shù)記錄需要截幾幀。
void TriggerCapture(uint32_t numFrames) { m_Cap = numFrames; }
??同文件下又提供了另一個方法bool RenderDoc::ShouldTriggerCapture(uint32_t frameNumber)抵卫。
bool RenderDoc::ShouldTriggerCapture(uint32_t frameNumber)
{
bool ret = m_Cap > 0;
if(m_Cap > 0)
m_Cap--;
//something....
return ret;
??繼續(xù)查找引用可以找到一個相對有用的函數(shù)——WrappedID3D11Device::Present狮荔，這個函數(shù)在結尾調(diào)用的ShouldTriggerCapture胎撇，并且這個函數(shù)中還可以調(diào)整OverlayText(RenderDoc左上角的覆蓋層)。
//在WrappedID3D11Device::Present中
if(IsBackgroundCapturing(m_State) && RenderDoc::Inst().ShouldTriggerCapture(m_FrameCounter))
{
RenderDoc::Inst().StartFrameCapture((ID3D11Device *)this, swapper->GetHWND());
m_AppControlledCapture = false;
m_CapturedFrames.back().frameNumber = m_FrameCounter;
}
void RenderDoc::StartFrameCapture(void *dev, void *wnd)
{
IFrameCapturer *frameCap = MatchFrameCapturer(dev, wnd);
if(frameCap)
{
frameCap->StartFrameCapture(dev, wnd);
m_CapturesActive++;
}
}
??frameCap是IFrameCapturer的指針殖氏，IFrameCapturer是個接口晚树，而實現(xiàn)這個接口的是WrappedID3D11Device。
void WrappedID3D11Device::StartFrameCapture(void *dev, void *wnd)
{
//something...
m_State = CaptureState::ActiveCapturing;
//something...
}
??在這里激活了當前抓取狀態(tài)雅采，core.h中封裝了一層函數(shù)：
constexpr inline bool IsActiveCapturing(CaptureState state)
{
return state == CaptureState::ActiveCapturing;
}
??當程序調(diào)用渲染api時脉执，會進入到RenderDoc定義的Hook函數(shù)中踏拜，在調(diào)用原有函數(shù)后档悠，會檢查截幀狀態(tài)派任，假如處于截幀狀態(tài)蔬墩，則記錄API睛竣，這就是抓幀的實現(xiàn)翻诉，貼一個例子还绘，下面是一個Wrap(包裝)好的api：
void WrappedID3D11DeviceContext::IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY Topology)
{
SCOPED_LOCK_OPTIONAL(m_pDevice->D3DLock(), m_pDevice->D3DThreadSafe());
DrainAnnotationQueue();
m_EmptyCommandList = false;
//調(diào)用原有函數(shù)
SERIALISE_TIME_CALL(m_pRealContext->IASetPrimitiveTopology(Topology));
//如果處于截幀狀態(tài)铐望，則序列化API和參數(shù)
if(IsActiveCapturing(m_State))
{
USE_SCRATCH_SERIALISER();
SCOPED_SERIALISE_CHUNK(D3D11Chunk::IASetPrimitiveTopology);
SERIALISE_ELEMENT(m_ResourceID).Named("Context"_lit).TypedAs("ID3D11DeviceContext *"_lit);
Serialise_IASetPrimitiveTopology(GET_SERIALISER, Topology);
m_ContextRecord->AddChunk(scope.Get());
}
m_CurrentPipelineState->Change(m_CurrentPipelineState->IA.Topo, Topology);
VerifyState();
}
??借此我們對這些Wrap的函數(shù)產(chǎn)生興趣冈涧，例如WrappedID3D11Device，查看聲明：
class WrappedID3D11Device : public IFrameCapturer, public ID3DDevice, public ID3D11Device5
??這個類竟然實現(xiàn)了ID3DDevice這個DX接口正蛙。
??斷點跟蹤下這個類的構造函數(shù)調(diào)用堆棧
-D3D11Hook::D3D11CreateDevice_hook
-D3D11Hook::D3D11CreateDeviceAndSwapChain_hook
-D3D11Hook::Create_Internal
-WrappedID3D11Device::WrappedID3D11Device
??第一個函數(shù)D3D11Hook::D3D11CreateDevice_hook是RenderDoc的hook函數(shù)督弓，本文的第一節(jié)已經(jīng)提到了；在D3D11Hook::Create_Internal中乒验，調(diào)用了原始的D3D11CreateDeviceAndSwapChain方法愚隧，然后用返回的device構造wrap device，最后將wrap返回锻全，由于wrap device同樣實現(xiàn)了ID3D11Device類的接口狂塘，因此在使用上和DirectX的Device沒有區(qū)別：
HRESULT Create_Internal(RealD3D11CreateFunction real, __in_opt IDXGIAdapter *pAdapter,
D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
__in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels,
UINT FeatureLevels, UINT SDKVersion,
__in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc,
__out_opt IDXGISwapChain **ppSwapChain, __out_opt ID3D11Device **ppDevice,
__out_opt D3D_FEATURE_LEVEL *pFeatureLevel,
__out_opt ID3D11DeviceContext **ppImmediateContext)
{
//...
//調(diào)用原有D3D11CreateDeviceAndSwapChain方法
HRESULT ret = real(pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels,
SDKVersion, pUsedSwapDesc, ppSwapChain, ppDevice, pFeatureLevel, NULL);
//構造wrap device
WrappedID3D11Device *wrap = new WrappedID3D11Device(*ppDevice, params);
//將wrap device的地址通過指針返回
*ppDevice = wrap;
//...
}
??其他DX組件，例如IDXGIFactory鳄厌、ID3D11DeviceContext的hook方式與此類似荞胡。
坑
??報個坑，Nvidia 驅(qū)動安裝程序Geforce Experience的游戲內(nèi)覆蓋如果開啟了嚎，有幾率會彈NVAPI_NO_IMPLEMENT錯誤泪漂。
