Refactor D3DX11 hook + implement ResizeBuffer handler

Destroy/release objects and recreate render view target on resolution change of the application
This commit is contained in:
Amos 2021-06-16 16:12:53 -07:00
parent 4db41e5e04
commit d424c75081
6 changed files with 259 additions and 77 deletions

81
r5dev/include/enums.h Normal file
View File

@ -0,0 +1,81 @@
#pragma once
/* Enumerations */
enum class D3D11DeviceVTbl : short
{
// IUnknown
QueryInterface = 0,
AddRef = 1,
Release = 2,
// ID3D11Device
CreateBuffer = 3,
CreateTexture1D = 4,
CreateTexture2D = 5,
CreateTexture3D = 6,
CreateShaderResourceView = 7,
CreateUnorderedAccessView = 8,
CreateRenderTargetView = 9,
CreateDepthStencilView = 10,
CreateInputLayout = 11,
CreateVertexShader = 12,
CreateGeometryShader = 13,
CreateGeometryShaderWithStreamOutput = 14,
CreatePixelShader = 15,
CreateHullShader = 16,
CreateDomainShader = 17,
CreateComputeShader = 18,
CreateClassLinkage = 19,
CreateBlendState = 20,
CreateDepthStencilState = 21,
CreateRasterizerState = 22,
CreateSamplerState = 23,
CreateQuery = 24,
CreatePredicate = 25,
CreateCounter = 26,
CreateDeferredContext = 27,
OpenSharedResource = 28,
CheckFormatSupport = 29,
CheckMultisampleQualityLevels = 30,
CheckCounterInfo = 31,
CheckCounter = 32,
CheckFeatureSupport = 33,
GetPrivateData = 34,
SetPrivateData = 35,
SetPrivateDataInterface = 36,
GetFeatureLevel = 37,
GetCreationFlags = 38,
GetDeviceRemovedReason = 39,
GetImmediateContext = 40,
SetExceptionMode = 41,
GetExceptionMode = 42,
};
enum class DXGISwapChainVTbl : short
{
// IUnknown
QueryInterface = 0,
AddRef = 1,
Release = 2,
// IDXGIObject
SetPrivateData = 3,
SetPrivateDataInterface = 4,
GetPrivateData = 5,
GetParent = 6,
// IDXGIDeviceSubObject
GetDevice = 7,
// IDXGISwapChain
Present = 8,
GetBuffer = 9,
SetFullscreenState = 10,
GetFullscreenState = 11,
GetDesc = 12,
ResizeBuffers = 13,
ResizeTarget = 14,
GetContainingOutput = 15,
GetFrameStatistics = 16,
GetLastPresentCount = 17,
};

View File

@ -1,8 +1,24 @@
#pragma once
#include <d3d11.h>
/////////////////////////////////////////////////////////////////////////////
// Initialization
void SetupImGui();
void SetupDXSwapChain();
void DrawImGui();
void DestroyRenderTarget();
/////////////////////////////////////////////////////////////////////////////
// Management
void PrintDXAddress();
void InstallDXHooks();
void RemoveDXHooks();
void SetupDXSwapChain();
/////////////////////////////////////////////////////////////////////////////
// Handlers
extern LRESULT ImGui_ImplWin32_WndProcHandler(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam);
extern HRESULT __stdcall Present(IDXGISwapChain* pSwapChain, UINT nSyncInterval, UINT nFlags);
/////////////////////////////////////////////////////////////////////////////
// Globals
extern DWORD g_dThreadId;

View File

@ -1,6 +1,6 @@
#pragma once
// Define structures/types
/* Structures */
typedef unsigned __int64 QWORD;
struct __declspec(align(8)) netpacket_t

View File

@ -193,6 +193,7 @@
<ClInclude Include="..\imgui\include\imstb_textedit.h" />
<ClInclude Include="..\imgui\include\imstb_truetype.h" />
<ClInclude Include="include\console.h" />
<ClInclude Include="include\enums.h" />
<ClInclude Include="include\hooks.h" />
<ClInclude Include="include\id3dx.h" />
<ClInclude Include="include\opcptc.h" />

View File

@ -137,6 +137,9 @@
<ClInclude Include="include\overlay.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\enums.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="r5dev.def">

View File

@ -6,6 +6,7 @@
#include <windows.h>
#include "id3dx.h"
#include "enums.h"
#include "detours.h"
#include "overlay.h"
#include "patterns.h"
@ -16,42 +17,45 @@
#pragma comment(lib, "d3d11.lib")
/*-----------------------------------------------------------------------------
/*---------------------------------------------------------------------------------
* _id3dx.cpp
*-----------------------------------------------------------------------------*/
*---------------------------------------------------------------------------------*/
/////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////
typedef HRESULT(__stdcall* IDXGISwapChainPresent)(IDXGISwapChain* pSwapChain, UINT nSyncInterval, UINT nFlags);
HRESULT __stdcall Present(IDXGISwapChain* pSwapChain, UINT nSyncInterval, UINT nFlags);
typedef HRESULT(__stdcall* IDXGIResizeBuffers) (IDXGISwapChain* pSwapChain, UINT nBufferCount, UINT nWidth, UINT nHeight, DXGI_FORMAT dxFormat, UINT nSwapChainFlags);
extern LRESULT CALLBACK DXGIMsgProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { return DefWindowProc(hwnd, uMsg, wParam, lParam); }
extern LRESULT ImGui_ImplWin32_WndProcHandler(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam);
///////////////////////////////////////////////////////////////////////////////////
static BOOL g_bShowMenu = false;
static BOOL g_bInitialized = false;
static BOOL g_bPresentHooked = false;
static BOOL g_bListener[256] = { 0 };
/////////////////////////////////////////////////////////////////////////////
static BOOL g_bShowMenu = false;
static BOOL g_bInitialized = false;
static BOOL g_bPresentHooked = false;
static HWND g_hGameWindow = NULL;
static WNDPROC g_oWndProc = NULL;
extern DWORD g_dThreadId = NULL;
static HWND g_hGameWindow = NULL;
static WNDPROC g_oWndProc = NULL;
/////////////////////////////////////////////////////////////////////////////
static IDXGISwapChainPresent g_fnIDXGISwapChainPresent = nullptr;
static IDXGISwapChain* g_pSwapChain = nullptr;
static ID3D11DeviceContext* g_pDeviceContext = nullptr;
static ID3D11Device* g_pDevice = nullptr;
static ID3D11RenderTargetView* g_pRenderTargetView = nullptr;
DWORD g_dThreadId = NULL;
///////////////////////////////////////////////////////////////////////////////////
static IDXGISwapChainPresent g_fnIDXGISwapChainPresent = nullptr;
static IDXGISwapChain* g_pSwapChain = nullptr;
static IDXGIResizeBuffers g_oResizeBuffers = nullptr;
static ID3D11DeviceContext* g_pDeviceContext = nullptr;
static ID3D11Device* g_pDevice = nullptr;
static ID3D11RenderTargetView* g_pRenderTargetView = nullptr;
//---------------------------------------------------------------------------------
// Window
//---------------------------------------------------------------------------------
void Hook_SetCursorPosition(int a1, INT64 posX, INT64 posY)
void Hook_SetCursorPosition(INT64 nFlag, LONG posX, LONG posY)
{
if (g_bShowMenu) { return; }
return SetCursorPosition(a1, posX, posY);
return SetCursorPosition(nFlag, posX, posY);
}
LRESULT CALLBACK DXGIMsgProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
{
return DefWindowProc(hWnd, uMsg, wParam, lParam);
}
LRESULT CALLBACK hWndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
@ -65,13 +69,17 @@ LRESULT CALLBACK hWndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
g_bShowMenu = !g_bShowMenu;
}
}
if (uMsg == WM_SIZE)
{
g_bShowMenu = false;
}
if (g_bShowMenu)
{
ImGui_ImplWin32_WndProcHandler(hWnd, uMsg, wParam, lParam);
return true;
}
///////////////////////////////////////////////////////////////////////////////////
return CallWindowProc(g_oWndProc, hWnd, uMsg, wParam, lParam);
}
@ -82,7 +90,10 @@ LRESULT CALLBACK hWndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
HRESULT GetDeviceAndCtxFromSwapchain(IDXGISwapChain* pSwapChain, ID3D11Device** ppDevice, ID3D11DeviceContext** ppContext)
{
HRESULT ret = pSwapChain->GetDevice(__uuidof(ID3D11Device), (PVOID*)ppDevice);
if (SUCCEEDED(ret)) { (*ppDevice)->GetImmediateContext(ppContext); }
if (SUCCEEDED(ret))
{
(*ppDevice)->GetImmediateContext(ppContext);
}
return ret;
}
@ -90,29 +101,30 @@ void GetPresent()
{
WNDCLASSEXA wc = { sizeof(WNDCLASSEX), CS_CLASSDC, DXGIMsgProc, 0L, 0L, GetModuleHandleA(NULL), NULL, NULL, NULL, NULL, "DX", NULL };
RegisterClassExA(&wc);
HWND hWnd = CreateWindowA("DX", NULL, WS_OVERLAPPEDWINDOW, 100, 100, 300, 300, NULL, NULL, wc.hInstance, NULL);
HWND hWnd = CreateWindowA("DX", NULL, WS_OVERLAPPEDWINDOW, 100, 100, 300, 300, NULL, NULL, wc.hInstance, NULL);
DXGI_SWAP_CHAIN_DESC sd;
D3D_FEATURE_LEVEL nFeatureLevelsSet = D3D_FEATURE_LEVEL_11_0;
D3D_FEATURE_LEVEL nFeatureLevelsSupported;
D3D_FEATURE_LEVEL nFeatureLevelsSet = D3D_FEATURE_LEVEL_11_0;
D3D_FEATURE_LEVEL nFeatureLevelsSupported;
ZeroMemory(&sd, sizeof(sd));
/////////////////////////////////////////////////////////////////////////////
sd.BufferCount = 1;
sd.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
sd.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
sd.BufferDesc.Height = 800;
sd.BufferDesc.Width = 600;
sd.BufferDesc.RefreshRate = { 60, 1 };
sd.BufferDesc.ScanlineOrdering = DXGI_MODE_SCANLINE_ORDER_UNSPECIFIED;
sd.BufferDesc.Scaling = DXGI_MODE_SCALING_UNSPECIFIED;
sd.Windowed = TRUE;
sd.OutputWindow = hWnd;//GetForegroundWindow();
sd.SampleDesc.Count = 1;
sd.SampleDesc.Quality = 0;
sd.SwapEffect = DXGI_SWAP_EFFECT_DISCARD;
///////////////////////////////////////////////////////////////////////////////
sd.BufferCount = 1;
sd.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
sd.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
sd.BufferDesc.Height = 800;
sd.BufferDesc.Width = 600;
sd.BufferDesc.RefreshRate = { 60, 1 };
sd.BufferDesc.ScanlineOrdering = DXGI_MODE_SCANLINE_ORDER_UNSPECIFIED;
sd.BufferDesc.Scaling = DXGI_MODE_SCALING_UNSPECIFIED;
sd.Windowed = TRUE;
sd.OutputWindow = hWnd;
sd.SampleDesc.Count = 1;
sd.SampleDesc.Quality = 0;
sd.SwapEffect = DXGI_SWAP_EFFECT_DISCARD;
/////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
g_hGameWindow = sd.OutputWindow;
UINT nFeatureLevelsRequested = 1;
HRESULT hr = 0;
@ -120,7 +132,7 @@ void GetPresent()
ID3D11Device* pDevice = nullptr;
ID3D11DeviceContext* pContext = nullptr;
/////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
if (FAILED(hr = D3D11CreateDeviceAndSwapChain(NULL,
D3D_DRIVER_TYPE_HARDWARE,
NULL,
@ -134,30 +146,37 @@ void GetPresent()
&nFeatureLevelsSupported,
&pContext)))
{
std::cout << "VMT hook failed" << std::endl;
std::cout << "+--------------------------------------------------------+" << std::endl;
std::cout << "| >>>>>>>>>| VIRTUAL METHOD TABLE HOOK FAILED |<<<<<<<<< |" << std::endl;
std::cout << "+--------------------------------------------------------+" << std::endl;
RemoveDXHooks();
return;
}
/////////////////////////////////////////////////////////////////////////////
DWORD_PTR* pSwapChainVtable = nullptr;
DWORD_PTR* pContextVTable = nullptr;
DWORD_PTR* pDeviceVTable = nullptr;
///////////////////////////////////////////////////////////////////////////////
DWORD_PTR* pSwapChainVtable = nullptr;
DWORD_PTR* pContextVTable = nullptr;
DWORD_PTR* pDeviceVTable = nullptr;
pSwapChainVtable = (DWORD_PTR*)pSwapChain;
pSwapChainVtable = (DWORD_PTR*)pSwapChainVtable[0];
pContextVTable = (DWORD_PTR*)pContext;
pContextVTable = (DWORD_PTR*)pContextVTable[0];
pDeviceVTable = (DWORD_PTR*)pDevice;
pDeviceVTable = (DWORD_PTR*)pDeviceVTable[0];
pSwapChainVtable = (DWORD_PTR*)pSwapChain;
pSwapChainVtable = (DWORD_PTR*)pSwapChainVtable[0];
pContextVTable = (DWORD_PTR*)pContext;
pContextVTable = (DWORD_PTR*)pContextVTable[0];
pDeviceVTable = (DWORD_PTR*)pDevice;
pDeviceVTable = (DWORD_PTR*)pDeviceVTable[0];
g_fnIDXGISwapChainPresent = (IDXGISwapChainPresent)(DWORD_PTR)pSwapChainVtable[8];
int pIDX = (int)DXGISwapChainVTbl::Present;
int rIDX = (int)DXGISwapChainVTbl::ResizeBuffers;
g_fnIDXGISwapChainPresent = (IDXGISwapChainPresent)(DWORD_PTR)pSwapChainVtable[pIDX];
g_oResizeBuffers = (IDXGIResizeBuffers)(DWORD_PTR)pSwapChainVtable[rIDX];
pSwapChain->Release();
pContext->Release();
pDevice->Release();
g_bPresentHooked = true;
///////////////////////////////////////////////////////////////////////////////
g_bPresentHooked = true;
}
//---------------------------------------------------------------------------------
@ -195,27 +214,77 @@ void DrawImGui()
ImGui_ImplDX11_RenderDrawData(ImGui::GetDrawData());
}
void RenderTarget(IDXGISwapChain* pSwapChain)
void CreateRenderTarget(IDXGISwapChain* pSwapChain)
{
/////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
DXGI_SWAP_CHAIN_DESC sd = { 0 };
ID3D11Texture2D* pBackBuffer = { 0 };
D3D11_RENDER_TARGET_VIEW_DESC render_target_view_desc;
/////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
pSwapChain->GetDesc(&sd);
ZeroMemory(&render_target_view_desc, sizeof(render_target_view_desc));
g_hGameWindow = sd.OutputWindow;
render_target_view_desc.Format = sd.BufferDesc.Format;
render_target_view_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2D;
g_hGameWindow = sd.OutputWindow;
render_target_view_desc.Format = sd.BufferDesc.Format;
render_target_view_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2D;
/////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
pSwapChain->GetBuffer(0, __uuidof(ID3D11Texture2D), (LPVOID*)&pBackBuffer);
if (pBackBuffer != NULL) { g_pDevice->CreateRenderTargetView(pBackBuffer, &render_target_view_desc, &g_pRenderTargetView); }
g_pDeviceContext->OMSetRenderTargets(1, &g_pRenderTargetView, NULL);
pBackBuffer->Release();
}
void CreateViewPort( UINT nWidth, UINT nHeight)
{
float width = *(float*)(&nWidth);
float height = *(float*)(&nHeight);
D3D11_VIEWPORT vp;
///////////////////////////////////////////////////////////////////////////////
vp.Width = width;
vp.Height = height;
vp.MinDepth = 0.0f;
vp.MaxDepth = 1.0f;
vp.TopLeftX = 0;
vp.TopLeftY = 0;
///////////////////////////////////////////////////////////////////////////////
g_pDeviceContext->RSSetViewports(1, &vp);
}
void DestroyRenderTarget()
{
if (nullptr != g_pRenderTargetView)
{
g_pRenderTargetView->Release();
g_pRenderTargetView = nullptr;
g_pDeviceContext->OMSetRenderTargets(0, 0, 0);
std::cout << "+--------------------------------------------------------+" << std::endl;
std::cout << "| >>>>>>>>>>>>>>| RENDER TARGET DESTORYED |<<<<<<<<<<<<< |" << std::endl;
std::cout << "+--------------------------------------------------------+" << std::endl;
}
}
static bool test;
HRESULT __stdcall GetResizeBuffers(IDXGISwapChain* pSwapChain, UINT nBufferCount, UINT nWidth, UINT nHeight, DXGI_FORMAT dxFormat, UINT nSwapChainFlags)
{
g_bShowMenu = false;
g_bInitialized = false;
g_bPresentHooked = false;
///////////////////////////////////////////////////////////////////////////////
ImGui_ImplDX11_InvalidateDeviceObjects();
DestroyRenderTarget();
CreateViewPort(nWidth, nHeight);
ImGui_ImplDX11_CreateDeviceObjects();
///////////////////////////////////////////////////////////////////////////////
return g_oResizeBuffers(pSwapChain, nBufferCount, nWidth, nHeight, dxFormat, nSwapChainFlags);
}
HRESULT __stdcall Present(IDXGISwapChain* pSwapChain, UINT nSyncInterval, UINT nFlags)
{
if (!g_bInitialized)
@ -223,19 +292,26 @@ HRESULT __stdcall Present(IDXGISwapChain* pSwapChain, UINT nSyncInterval, UINT n
if (FAILED(GetDeviceAndCtxFromSwapchain(pSwapChain, &g_pDevice, &g_pDeviceContext)))
{
return g_fnIDXGISwapChainPresent(pSwapChain, nSyncInterval, nFlags);
std::cout << "+--------------------------------------------------------+" << std::endl;
std::cout << "| >>>>>>>>>>| GET DVS AND CTX FROM SCP FAILED |<<<<<<<<< |" << std::endl;
std::cout << "+--------------------------------------------------------+" << std::endl;
}
RenderTarget(pSwapChain);
CreateRenderTarget(pSwapChain);
SetupImGui();
g_oWndProc = (WNDPROC)SetWindowLongPtr(g_hGameWindow, GWLP_WNDPROC, (LONG_PTR)hWndProc);
if (g_oWndProc == nullptr)
{ // Only initialize hWndProc pointer once to avoid stack overflow during ResizeBuffers(..)
g_oWndProc = (WNDPROC)SetWindowLongPtr(g_hGameWindow, GWLP_WNDPROC, (LONG_PTR)hWndProc);
}
g_bInitialized = true;
g_pSwapChain = pSwapChain;
}
DrawImGui();
g_bInitialized = true;
/////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
return g_fnIDXGISwapChainPresent(pSwapChain, nSyncInterval, nFlags);
}
@ -249,6 +325,7 @@ void InstallDXHooks()
DetourUpdateThread(GetCurrentThread());
DetourAttach((LPVOID*)&SetCursorPosition, &Hook_SetCursorPosition);
DetourAttach(&(LPVOID&)g_fnIDXGISwapChainPresent, (PBYTE)Present);
DetourAttach(&(LPVOID&)g_oResizeBuffers, (PBYTE)GetResizeBuffers);
DetourTransactionCommit();
}
@ -258,18 +335,23 @@ void RemoveDXHooks()
DetourUpdateThread(GetCurrentThread());
DetourDetach((LPVOID*)&SetCursorPosition, &Hook_SetCursorPosition);
DetourDetach(&(LPVOID&)g_fnIDXGISwapChainPresent, (PBYTE)Present);
DetourDetach(&(LPVOID&)g_oResizeBuffers, (PBYTE)GetResizeBuffers);
DetourTransactionCommit();
///////////////////////////////////////////////////////////////////////////////
ImGui_ImplWin32_Shutdown();
ImGui_ImplDX11_Shutdown();
}
void PrintDXAddress()
{
std::cout << "--------------------------------------------------" << std::endl;
std::cout << " ID3D11DeviceContext : " << std::hex << g_pDeviceContext << std::endl;
std::cout << " ID3D11Device : " << std::hex << g_pDevice << std::endl;
std::cout << " ID3D11RenderTargetView : " << std::hex << g_pRenderTargetView << std::endl;
std::cout << " IDXGISwapChain : " << std::hex << g_pSwapChain << std::endl;
std::cout << " IDXGISwapChainPresent : " << std::hex << g_fnIDXGISwapChainPresent << std::endl;
std::cout << "--------------------------------------------------" << std::endl;
std::cout << "+--------------------------------------------------------+" << std::endl;
std::cout << "| ID3D11DeviceContext : " << std::hex << g_pDeviceContext << std::endl;
std::cout << "| ID3D11Device : " << std::hex << g_pDevice << std::endl;
std::cout << "| ID3D11RenderTargetView : " << std::hex << g_pRenderTargetView << std::endl;
std::cout << "| IDXGISwapChain : " << std::hex << g_pSwapChain << std::endl;
std::cout << "| IDXGISwapChainPresent : " << std::hex << g_fnIDXGISwapChainPresent << std::endl;
std::cout << "+--------------------------------------------------------+" << std::endl;
}
//---------------------------------------------------------------------------------
@ -280,7 +362,6 @@ DWORD __stdcall DXSwapChainWorker(LPVOID)
{
GetPresent();
InstallDXHooks();
if (!g_bPresentHooked) { ImGui_ImplWin32_Shutdown(); ImGui_ImplDX11_Shutdown(); RemoveDXHooks(); }
return true;
}