Refactor D3DX11 hook + implement ResizeBuffer handler

Destroy/release objects and recreate render view target on resolution change of the application

r5dev/include/enums.h

@@ -0,0 +1,81 @@
+#pragma once
+
+/* Enumerations */
+enum class D3D11DeviceVTbl : short
+{
+    // IUnknown
+    QueryInterface                       = 0,
+    AddRef                               = 1,
+    Release                              = 2,
+
+    // ID3D11Device
+    CreateBuffer                         = 3,
+    CreateTexture1D                      = 4,
+    CreateTexture2D                      = 5,
+    CreateTexture3D                      = 6,
+    CreateShaderResourceView             = 7,
+    CreateUnorderedAccessView            = 8,
+    CreateRenderTargetView               = 9,
+    CreateDepthStencilView               = 10,
+    CreateInputLayout                    = 11,
+    CreateVertexShader                   = 12,
+    CreateGeometryShader                 = 13,
+    CreateGeometryShaderWithStreamOutput = 14,
+    CreatePixelShader                    = 15,
+    CreateHullShader                     = 16,
+    CreateDomainShader                   = 17,
+    CreateComputeShader                  = 18,
+    CreateClassLinkage                   = 19,
+    CreateBlendState                     = 20,
+    CreateDepthStencilState              = 21,
+    CreateRasterizerState                = 22,
+    CreateSamplerState                   = 23,
+    CreateQuery                          = 24,
+    CreatePredicate                      = 25,
+    CreateCounter                        = 26,
+    CreateDeferredContext                = 27,
+    OpenSharedResource                   = 28,
+    CheckFormatSupport                   = 29,
+    CheckMultisampleQualityLevels        = 30,
+    CheckCounterInfo                     = 31,
+    CheckCounter                         = 32,
+    CheckFeatureSupport                  = 33,
+    GetPrivateData                       = 34,
+    SetPrivateData                       = 35,
+    SetPrivateDataInterface              = 36,
+    GetFeatureLevel                      = 37,
+    GetCreationFlags                     = 38,
+    GetDeviceRemovedReason               = 39,
+    GetImmediateContext                  = 40,
+    SetExceptionMode                     = 41,
+    GetExceptionMode                     = 42,
+};
+
+enum class DXGISwapChainVTbl : short
+{
+    // IUnknown
+    QueryInterface                       = 0,
+    AddRef                               = 1,
+    Release                              = 2,
+
+    // IDXGIObject
+    SetPrivateData                       = 3,
+    SetPrivateDataInterface              = 4,
+    GetPrivateData                       = 5,
+    GetParent                            = 6,
+
+    // IDXGIDeviceSubObject
+    GetDevice                            = 7,
+
+    // IDXGISwapChain
+    Present                              = 8,
+    GetBuffer                            = 9,
+    SetFullscreenState                   = 10,
+    GetFullscreenState                   = 11,
+    GetDesc                              = 12,
+    ResizeBuffers                        = 13,
+    ResizeTarget                         = 14,
+    GetContainingOutput                  = 15,
+    GetFrameStatistics                   = 16,
+    GetLastPresentCount                  = 17,
+};

r5dev/include/id3dx.h

@@ -1,8 +1,24 @@
 #pragma once
+#include <d3d11.h>
 
+/////////////////////////////////////////////////////////////////////////////
+// Initialization
+void SetupImGui();
+void SetupDXSwapChain();
+void DrawImGui();
+void DestroyRenderTarget();
+
+/////////////////////////////////////////////////////////////////////////////
+// Management
 void PrintDXAddress();
 void InstallDXHooks();
 void RemoveDXHooks();
-void SetupDXSwapChain();
 
+/////////////////////////////////////////////////////////////////////////////
+// Handlers
+extern LRESULT ImGui_ImplWin32_WndProcHandler(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam);
+extern HRESULT __stdcall Present(IDXGISwapChain* pSwapChain, UINT nSyncInterval, UINT nFlags);
+
+/////////////////////////////////////////////////////////////////////////////
+// Globals
 extern DWORD g_dThreadId;

r5dev/include/structs.h

@@ -1,6 +1,6 @@
 #pragma once
 
-// Define structures/types
+/* Structures */
 typedef unsigned __int64 QWORD;
 
 struct __declspec(align(8)) netpacket_t

r5dev/r5dev.vcxproj

@@ -193,6 +193,7 @@
     <ClInclude Include="..\imgui\include\imstb_textedit.h" />
     <ClInclude Include="..\imgui\include\imstb_truetype.h" />
     <ClInclude Include="include\console.h" />
+    <ClInclude Include="include\enums.h" />
     <ClInclude Include="include\hooks.h" />
     <ClInclude Include="include\id3dx.h" />
     <ClInclude Include="include\opcptc.h" />

r5dev/r5dev.vcxproj.filters

@@ -137,6 +137,9 @@
     <ClInclude Include="include\overlay.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="include\enums.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="r5dev.def">

r5dev/src/id3dx.cpp

@@ -6,6 +6,7 @@
 #include <windows.h>
 
 #include "id3dx.h"
+#include "enums.h"
 #include "detours.h"
 #include "overlay.h"
 #include "patterns.h"
@@ -16,42 +17,45 @@
 
 #pragma comment(lib, "d3d11.lib")
 
-/*-----------------------------------------------------------------------------
+/*---------------------------------------------------------------------------------
  * _id3dx.cpp
- *-----------------------------------------------------------------------------*/
+ *---------------------------------------------------------------------------------*/
 
-/////////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////////
 typedef HRESULT(__stdcall* IDXGISwapChainPresent)(IDXGISwapChain* pSwapChain, UINT nSyncInterval, UINT nFlags);
-HRESULT __stdcall Present(IDXGISwapChain* pSwapChain, UINT nSyncInterval, UINT nFlags);
+typedef HRESULT(__stdcall* IDXGIResizeBuffers)   (IDXGISwapChain* pSwapChain, UINT nBufferCount, UINT nWidth, UINT nHeight, DXGI_FORMAT dxFormat, UINT nSwapChainFlags);
 
-extern LRESULT CALLBACK DXGIMsgProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { return DefWindowProc(hwnd, uMsg, wParam, lParam); }
+///////////////////////////////////////////////////////////////////////////////////
-extern LRESULT ImGui_ImplWin32_WndProcHandler(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam);
-
-/////////////////////////////////////////////////////////////////////////////
 static BOOL g_bShowMenu         = false;
 static BOOL g_bInitialized      = false;
 static BOOL g_bPresentHooked    = false;
+static BOOL g_bListener[256]    = { 0 };
 
 static HWND     g_hGameWindow   = NULL;
 static WNDPROC  g_oWndProc      = NULL;
+extern DWORD    g_dThreadId     = NULL;
 
-/////////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////////
 static IDXGISwapChainPresent    g_fnIDXGISwapChainPresent   = nullptr;
 static IDXGISwapChain*          g_pSwapChain                = nullptr;
+static IDXGIResizeBuffers       g_oResizeBuffers            = nullptr;
 static ID3D11DeviceContext*     g_pDeviceContext            = nullptr;
 static ID3D11Device*            g_pDevice                   = nullptr;
 static ID3D11RenderTargetView*  g_pRenderTargetView         = nullptr;
 
-DWORD g_dThreadId               = NULL;
-
 //---------------------------------------------------------------------------------
 // Window
 //---------------------------------------------------------------------------------
 
-void Hook_SetCursorPosition(int a1, INT64 posX, INT64 posY)
+void Hook_SetCursorPosition(INT64 nFlag, LONG posX, LONG posY)
 {
 	if (g_bShowMenu) { return; }
-	return SetCursorPosition(a1, posX, posY);
+	return SetCursorPosition(nFlag, posX, posY);
+}
+
+LRESULT CALLBACK DXGIMsgProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
+{
+	return DefWindowProc(hWnd, uMsg, wParam, lParam);
 }
 
 LRESULT CALLBACK hWndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
@@ -65,13 +69,17 @@ LRESULT CALLBACK hWndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
 			g_bShowMenu = !g_bShowMenu;
 		}
 	}
-
+	if (uMsg == WM_SIZE)
+	{
+		g_bShowMenu = false;
+	}
 	if (g_bShowMenu)
 	{
 		ImGui_ImplWin32_WndProcHandler(hWnd, uMsg, wParam, lParam);
 		return true;
 	}
 
+	///////////////////////////////////////////////////////////////////////////////////
 	return CallWindowProc(g_oWndProc, hWnd, uMsg, wParam, lParam);
 }
 
@@ -82,7 +90,10 @@ LRESULT CALLBACK hWndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
 HRESULT GetDeviceAndCtxFromSwapchain(IDXGISwapChain* pSwapChain, ID3D11Device** ppDevice, ID3D11DeviceContext** ppContext)
 {
 	HRESULT ret = pSwapChain->GetDevice(__uuidof(ID3D11Device), (PVOID*)ppDevice);
-	if (SUCCEEDED(ret)) { (*ppDevice)->GetImmediateContext(ppContext); }
+	if (SUCCEEDED(ret))
+	{
+		(*ppDevice)->GetImmediateContext(ppContext);
+	}
 	return ret;
 }
 
@@ -90,14 +101,15 @@ void GetPresent()
 {
 	WNDCLASSEXA wc = { sizeof(WNDCLASSEX), CS_CLASSDC, DXGIMsgProc, 0L, 0L, GetModuleHandleA(NULL), NULL, NULL, NULL, NULL, "DX", NULL };
 	RegisterClassExA(&wc);
-	HWND hWnd = CreateWindowA("DX", NULL, WS_OVERLAPPEDWINDOW, 100, 100, 300, 300, NULL, NULL, wc.hInstance, NULL);
 
+	HWND hWnd = CreateWindowA("DX", NULL, WS_OVERLAPPEDWINDOW, 100, 100, 300, 300, NULL, NULL, wc.hInstance, NULL);
 	DXGI_SWAP_CHAIN_DESC sd;
 	D3D_FEATURE_LEVEL    nFeatureLevelsSet = D3D_FEATURE_LEVEL_11_0;
 	D3D_FEATURE_LEVEL    nFeatureLevelsSupported;
+
 	ZeroMemory(&sd, sizeof(sd));
 
-	/////////////////////////////////////////////////////////////////////////////
+	///////////////////////////////////////////////////////////////////////////////
 	sd.BufferCount                          = 1;
 	sd.BufferUsage                          = DXGI_USAGE_RENDER_TARGET_OUTPUT;
 	sd.BufferDesc.Format                    = DXGI_FORMAT_R8G8B8A8_UNORM;
@@ -107,12 +119,12 @@ void GetPresent()
 	sd.BufferDesc.ScanlineOrdering          = DXGI_MODE_SCANLINE_ORDER_UNSPECIFIED;
 	sd.BufferDesc.Scaling                   = DXGI_MODE_SCALING_UNSPECIFIED;
 	sd.Windowed                             = TRUE;
-	sd.OutputWindow                 = hWnd;//GetForegroundWindow();
+	sd.OutputWindow                         = hWnd;
 	sd.SampleDesc.Count                     = 1;
 	sd.SampleDesc.Quality                   = 0;
 	sd.SwapEffect                           = DXGI_SWAP_EFFECT_DISCARD;
 
-	/////////////////////////////////////////////////////////////////////////////
+	///////////////////////////////////////////////////////////////////////////////
 	g_hGameWindow                           = sd.OutputWindow;
 	UINT       nFeatureLevelsRequested      = 1;
 	HRESULT                 hr              = 0;
@@ -120,7 +132,7 @@ void GetPresent()
 	ID3D11Device*           pDevice         = nullptr;
 	ID3D11DeviceContext*    pContext        = nullptr;
 
-	/////////////////////////////////////////////////////////////////////////////
+	///////////////////////////////////////////////////////////////////////////////
 	if (FAILED(hr = D3D11CreateDeviceAndSwapChain(NULL,
 		D3D_DRIVER_TYPE_HARDWARE,
 		NULL,
@@ -134,12 +146,14 @@ void GetPresent()
 		&nFeatureLevelsSupported,
 		&pContext)))
 	{
-		std::cout << "VMT hook failed" << std::endl;
+		std::cout << "+--------------------------------------------------------+" << std::endl;
+		std::cout << "| >>>>>>>>>| VIRTUAL METHOD TABLE HOOK FAILED |<<<<<<<<< |" << std::endl;
+		std::cout << "+--------------------------------------------------------+" << std::endl;
 		RemoveDXHooks();
 		return;
 	}
 
-	/////////////////////////////////////////////////////////////////////////////
+	///////////////////////////////////////////////////////////////////////////////
 	DWORD_PTR* pSwapChainVtable		        = nullptr;
 	DWORD_PTR* pContextVTable		        = nullptr;
 	DWORD_PTR* pDeviceVTable		        = nullptr;
@@ -151,12 +165,17 @@ void GetPresent()
 	pDeviceVTable			  = (DWORD_PTR*)pDevice;
 	pDeviceVTable			  = (DWORD_PTR*)pDeviceVTable[0];
 
-	g_fnIDXGISwapChainPresent = (IDXGISwapChainPresent)(DWORD_PTR)pSwapChainVtable[8];
+	int pIDX                  = (int)DXGISwapChainVTbl::Present;
+	int rIDX                  = (int)DXGISwapChainVTbl::ResizeBuffers;
+
+	g_fnIDXGISwapChainPresent = (IDXGISwapChainPresent)(DWORD_PTR)pSwapChainVtable[pIDX];
+	g_oResizeBuffers          = (IDXGIResizeBuffers)(DWORD_PTR)pSwapChainVtable[rIDX];
 
 	pSwapChain->Release();
 	pContext->Release();
 	pDevice->Release();
 
+	///////////////////////////////////////////////////////////////////////////////
 	g_bPresentHooked          = true;
 }
 
@@ -195,14 +214,14 @@ void DrawImGui()
 	ImGui_ImplDX11_RenderDrawData(ImGui::GetDrawData());
 }
 
-void RenderTarget(IDXGISwapChain* pSwapChain)
+void CreateRenderTarget(IDXGISwapChain* pSwapChain)
 {
-	/////////////////////////////////////////////////////////////////////////////
+	///////////////////////////////////////////////////////////////////////////////
 	DXGI_SWAP_CHAIN_DESC            sd          = { 0 };
 	ID3D11Texture2D*                pBackBuffer = { 0 };
 	D3D11_RENDER_TARGET_VIEW_DESC   render_target_view_desc;
 
-	/////////////////////////////////////////////////////////////////////////////
+	///////////////////////////////////////////////////////////////////////////////
 	pSwapChain->GetDesc(&sd);
 	ZeroMemory(&render_target_view_desc, sizeof(render_target_view_desc));
 
@@ -210,12 +229,62 @@ void RenderTarget(IDXGISwapChain* pSwapChain)
 	render_target_view_desc.Format = sd.BufferDesc.Format;
 	render_target_view_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2D;
 
-	/////////////////////////////////////////////////////////////////////////////
+	///////////////////////////////////////////////////////////////////////////////
 	pSwapChain->GetBuffer(0, __uuidof(ID3D11Texture2D), (LPVOID*)&pBackBuffer);
 	if (pBackBuffer != NULL) { g_pDevice->CreateRenderTargetView(pBackBuffer, &render_target_view_desc, &g_pRenderTargetView); }
+	g_pDeviceContext->OMSetRenderTargets(1, &g_pRenderTargetView, NULL);
 	pBackBuffer->Release();
 }
 
+void CreateViewPort( UINT nWidth, UINT nHeight)
+{
+	float width  = *(float*)(&nWidth);
+	float height = *(float*)(&nHeight);
+
+	D3D11_VIEWPORT vp;
+
+	///////////////////////////////////////////////////////////////////////////////
+	vp.Width     = width;
+	vp.Height    = height;
+	vp.MinDepth  = 0.0f;
+	vp.MaxDepth  = 1.0f;
+	vp.TopLeftX  = 0;
+	vp.TopLeftY  = 0;
+
+	///////////////////////////////////////////////////////////////////////////////
+	g_pDeviceContext->RSSetViewports(1, &vp);
+}
+
+void DestroyRenderTarget()
+{
+	if (nullptr != g_pRenderTargetView)
+	{
+		g_pRenderTargetView->Release();
+		g_pRenderTargetView = nullptr;
+		g_pDeviceContext->OMSetRenderTargets(0, 0, 0);
+		std::cout << "+--------------------------------------------------------+" << std::endl;
+		std::cout << "| >>>>>>>>>>>>>>| RENDER TARGET DESTORYED |<<<<<<<<<<<<< |" << std::endl;
+		std::cout << "+--------------------------------------------------------+" << std::endl;
+	}
+}
+static bool test;
+HRESULT __stdcall GetResizeBuffers(IDXGISwapChain* pSwapChain, UINT nBufferCount, UINT nWidth, UINT nHeight, DXGI_FORMAT dxFormat, UINT nSwapChainFlags)
+{
+	g_bShowMenu       = false;
+	g_bInitialized    = false;
+	g_bPresentHooked  = false;
+
+	///////////////////////////////////////////////////////////////////////////////
+	ImGui_ImplDX11_InvalidateDeviceObjects();
+
+	DestroyRenderTarget();
+	CreateViewPort(nWidth, nHeight);
+
+	ImGui_ImplDX11_CreateDeviceObjects();
+	///////////////////////////////////////////////////////////////////////////////
+	return g_oResizeBuffers(pSwapChain, nBufferCount, nWidth, nHeight, dxFormat, nSwapChainFlags);
+}
+
 HRESULT __stdcall Present(IDXGISwapChain* pSwapChain, UINT nSyncInterval, UINT nFlags)
 {
 	if (!g_bInitialized)
@@ -223,19 +292,26 @@ HRESULT __stdcall Present(IDXGISwapChain* pSwapChain, UINT nSyncInterval, UINT n
 		if (FAILED(GetDeviceAndCtxFromSwapchain(pSwapChain, &g_pDevice, &g_pDeviceContext)))
 		{
 			return g_fnIDXGISwapChainPresent(pSwapChain, nSyncInterval, nFlags);
+			std::cout << "+--------------------------------------------------------+" << std::endl;
+			std::cout << "| >>>>>>>>>>| GET DVS AND CTX FROM SCP FAILED |<<<<<<<<< |" << std::endl;
+			std::cout << "+--------------------------------------------------------+" << std::endl;
 		}
 
-		RenderTarget(pSwapChain);
+		CreateRenderTarget(pSwapChain);
 		SetupImGui();
 
+		if (g_oWndProc == nullptr)
+		{   // Only initialize hWndProc pointer once to avoid stack overflow during ResizeBuffers(..)
 			g_oWndProc  = (WNDPROC)SetWindowLongPtr(g_hGameWindow, GWLP_WNDPROC, (LONG_PTR)hWndProc);
+		}
+
 		g_bInitialized  = true;
 		g_pSwapChain    = pSwapChain;
 	}
 
 	DrawImGui();
 	g_bInitialized      = true;
-	/////////////////////////////////////////////////////////////////////////////
+	///////////////////////////////////////////////////////////////////////////////
 	return g_fnIDXGISwapChainPresent(pSwapChain, nSyncInterval, nFlags);
 }
 
@@ -249,6 +325,7 @@ void InstallDXHooks()
 	DetourUpdateThread(GetCurrentThread());
 	DetourAttach((LPVOID*)&SetCursorPosition, &Hook_SetCursorPosition);
 	DetourAttach(&(LPVOID&)g_fnIDXGISwapChainPresent, (PBYTE)Present);
+	DetourAttach(&(LPVOID&)g_oResizeBuffers, (PBYTE)GetResizeBuffers);
 	DetourTransactionCommit();
 }
 
@@ -258,18 +335,23 @@ void RemoveDXHooks()
 	DetourUpdateThread(GetCurrentThread());
 	DetourDetach((LPVOID*)&SetCursorPosition, &Hook_SetCursorPosition);
 	DetourDetach(&(LPVOID&)g_fnIDXGISwapChainPresent, (PBYTE)Present);
+	DetourDetach(&(LPVOID&)g_oResizeBuffers, (PBYTE)GetResizeBuffers);
 	DetourTransactionCommit();
+
+	///////////////////////////////////////////////////////////////////////////////
+	ImGui_ImplWin32_Shutdown();
+	ImGui_ImplDX11_Shutdown();
 }
 
 void PrintDXAddress()
 {
-	std::cout << "--------------------------------------------------" << std::endl;
+	std::cout << "+--------------------------------------------------------+" << std::endl;
-	std::cout << " ID3D11DeviceContext    : " << std::hex << g_pDeviceContext << std::endl;
+	std::cout << "| ID3D11DeviceContext    : " << std::hex << g_pDeviceContext << std::endl;
-	std::cout << " ID3D11Device           : " << std::hex << g_pDevice << std::endl;
+	std::cout << "| ID3D11Device           : " << std::hex << g_pDevice << std::endl;
-	std::cout << " ID3D11RenderTargetView : " << std::hex << g_pRenderTargetView << std::endl;
+	std::cout << "| ID3D11RenderTargetView : " << std::hex << g_pRenderTargetView << std::endl;
-	std::cout << " IDXGISwapChain         : " << std::hex << g_pSwapChain << std::endl;
+	std::cout << "| IDXGISwapChain         : " << std::hex << g_pSwapChain << std::endl;
-	std::cout << " IDXGISwapChainPresent  : " << std::hex << g_fnIDXGISwapChainPresent << std::endl;
+	std::cout << "| IDXGISwapChainPresent  : " << std::hex << g_fnIDXGISwapChainPresent << std::endl;
-	std::cout << "--------------------------------------------------" << std::endl;
+	std::cout << "+--------------------------------------------------------+" << std::endl;
 }
 
 //---------------------------------------------------------------------------------
@@ -280,7 +362,6 @@ DWORD __stdcall DXSwapChainWorker(LPVOID)
 {
 	GetPresent();
 	InstallDXHooks();
-	if (!g_bPresentHooked) { ImGui_ImplWin32_Shutdown(); ImGui_ImplDX11_Shutdown(); RemoveDXHooks(); }
 	return true;
 }