Implement LZHAM compression for the signature cache map

r5dev/public/utility/sigcache.cpp

@@ -5,7 +5,7 @@
 //===========================================================================//
 #include "core/stdafx.h"
 #include "public/utility/binstream.h"
-#include "sigcache.h"
+#include "public/utility/sigcache.h"
 
 //-----------------------------------------------------------------------------
 // Purpose: creates a pair of a pattern (key) and relative virtual address (value)
@@ -59,30 +59,44 @@ bool CSigCache::LoadCache(const string& svCacheFile)
 		return false;
 	}
 
-	SigDBHeader_t sigDbHeader;
+	SigDBHeader_t header;
-	sigDbHeader.m_nMagic = reader.Read<int>();
+	header.m_nMagic = reader.Read<int>();
 
-	if (sigDbHeader.m_nMagic != SIGDB_MAGIC)
+	if (header.m_nMagic != SIGDB_MAGIC)
 	{
 		return false;
 	}
 
-	sigDbHeader.m_nVersion = reader.Read<int>();
+	header.m_nMajorVersion = reader.Read<uint16_t>();
-	if (sigDbHeader.m_nVersion != SIGDB_VERSION)
+	if (header.m_nMajorVersion != SIGDB_MAJOR_VERSION)
 	{
 		return false;
 	}
 
-	sigDbHeader.m_FileTime = reader.Read<FILETIME>();
+	header.m_nMinorVersion = reader.Read<uint16_t>();
+	if (header.m_nMinorVersion != SIGDB_MINOR_VERSION)
+	{
+		return false;
+	}
 
-	vector<uint8_t> vData;
+	header.m_nBlobSizeMem = reader.Read<uint64_t>();
-	size_t nSize = (static_cast<size_t>(reader.GetSize()) - sizeof(SigDBHeader_t));
+	header.m_nBlobSizeDisk = reader.Read<uint64_t>();
+	header.m_nBlobHash = reader.Read<uint32_t>();
 
-	vData.resize(nSize);
+	uint32_t nCrc32;
-	uint8_t* pBuf = vData.data();
-	reader.Read<uint8_t>(*pBuf, nSize);
 
-	if (!m_Cache.ParseFromArray(pBuf, nSize))
+	std::unique_ptr<uint8_t[]> pSrcBuf(new uint8_t[header.m_nBlobSizeDisk]);
+	std::unique_ptr<uint8_t[]> pDstBuf(new uint8_t[header.m_nBlobSizeMem]);
+
+	reader.Read<uint8_t>(*pSrcBuf.get(), header.m_nBlobSizeDisk);
+	DecompressBlob(header.m_nBlobSizeDisk, header.m_nBlobSizeMem, nCrc32, pSrcBuf.get(), pDstBuf.get());
+
+	if (header.m_nBlobHash != nCrc32)
+	{
+		return false;
+	}
+
+	if (!m_Cache.ParseFromArray(pDstBuf.get(), header.m_nBlobSizeMem))
 	{
 		return false;
 	}
@@ -106,19 +120,84 @@ bool CSigCache::WriteCache(const string& svCacheFile)
 	CIOStream writer(svCacheFile, CIOStream::Mode_t::WRITE);
 	if (!writer.IsWritable())
 	{
-		// Error message..
+		Error(eDLL_T::COMMON, NO_ERROR, "Failed to write cache file: (read-only?)\n");
 		return false;
 	}
 
 	SigDBHeader_t header;
-
 	header.m_nMagic = SIGDB_MAGIC;
-	header.m_nVersion = SIGDB_VERSION;
+	header.m_nMajorVersion = SIGDB_MAJOR_VERSION;
-	GetSystemTimeAsFileTime(&header.m_FileTime);
+	header.m_nMinorVersion = SIGDB_MINOR_VERSION;
+
 	const string svBuffer = m_Cache.SerializeAsString();
+	std::unique_ptr<uint8_t[]> pBuffer(new uint8_t[svBuffer.size()]);
+
+	header.m_nBlobSizeMem = svBuffer.size();
+	uint64_t nCompSize = svBuffer.size();
+
+	if (!CompressBlob(svBuffer.size(), nCompSize, header.m_nBlobHash, reinterpret_cast<const uint8_t*>(svBuffer.data()), pBuffer.get()))
+	{
+		return false;
+	}
+
+	header.m_nBlobSizeDisk = nCompSize;
 
 	writer.Write(header);
-	writer.Write(svBuffer.data(), svBuffer.size());
+	writer.Write(pBuffer.get(), nCompSize);
+
+	return true;
+}
+
+//-----------------------------------------------------------------------------
+// Purpose: decompresses the blob containing the signature map
+// Input  : nSrcLen - 
+//			&nDstSize - 
+//			&nCrc32 - 
+//			*pSrcBuf - 
+//			*pDstBuf - 
+// Output : true on success, false otherwise
+//-----------------------------------------------------------------------------
+bool CSigCache::DecompressBlob(size_t nSrcLen, size_t& nDstLen, uint32_t& nCrc32, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const
+{
+	lzham_decompress_params lzDecompParams{};
+	lzDecompParams.m_dict_size_log2 = SIGDB_DICT_SIZE;
+	lzDecompParams.m_decompress_flags = lzham_decompress_flags::LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED | lzham_decompress_flags::LZHAM_DECOMP_FLAG_COMPUTE_CRC32;
+	lzDecompParams.m_struct_size = sizeof(lzham_decompress_params);
+
+	lzham_decompress_status_t lzDecompStatus = lzham_decompress_memory(&lzDecompParams, pDstBuf, &nDstLen, pSrcBuf, nSrcLen, NULL, &nCrc32);
+
+	if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS)
+	{
+		Error(eDLL_T::COMMON, NO_ERROR, "Failed to decompress blob: status = %08x\n", lzDecompStatus);
+		return false;
+	}
+
+	return true;
+}
+
+//-----------------------------------------------------------------------------
+// Purpose: compresses the blob containing the signature map
+// Input  : nSrcLen - 
+//			&nDstSize - 
+//			&nCrc32 - 
+//			*pSrcBuf - 
+//			*pDstBuf - 
+// Output : true on success, false otherwise
+//-----------------------------------------------------------------------------
+bool CSigCache::CompressBlob(size_t nSrcLen, size_t& nDstSize, uint32_t& nCrc32, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const
+{
+	lzham_compress_params lzCompParams{};
+	lzCompParams.m_dict_size_log2 = SIGDB_DICT_SIZE;
+	lzCompParams.m_level = lzham_compress_level::LZHAM_COMP_LEVEL_FASTEST;
+	lzCompParams.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING;
+
+	lzham_compress_status_t lzCompStatus = lzham_compress_memory(&lzCompParams, pDstBuf, &nDstSize, pSrcBuf, nSrcLen, NULL, &nCrc32);
+
+	if (lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
+	{
+		Warning(eDLL_T::COMMON, NO_ERROR, "Failed to compress blob: status = %08x\n", lzCompStatus);
+		return false;
+	}
 
 	return true;
 }

r5dev/public/utility/sigcache.h

@@ -4,7 +4,10 @@
 #include "protoc/sig_map.pb.h"
 
 #define SIGDB_MAGIC	(('p'<<24)+('a'<<16)+('M'<<8)+'S')
-#define SIGDB_VERSION 0x1
+#define SIGDB_DICT_SIZE 20
+
+#define SIGDB_MAJOR_VERSION 0x1 // Increment when library changes are made.
+#define SIGDB_MINOR_VERSION 0x1 // Increment when SDK updates are released.
 
 #ifdef DEDICATED
 #define SIGDB_FILE "cfg\\server\\startup.bin"
@@ -15,21 +18,33 @@
 class CSigCache
 {
 public:
+	CSigCache() { m_bInitialized = false; };
+	~CSigCache() {};
+
 	void AddEntry(const string& svPattern, const uint64_t nRVA);
 	bool FindEntry(const string& svPattern, uint64_t& nRVA) const;
 
 	bool LoadCache(const string& svCacheFile);
 	bool WriteCache(const string& svCacheFile);
 
+private:
+	bool CompressBlob(size_t nSrcLen, size_t& nDstSize, uint32_t& nCrc32, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const;
+	bool DecompressBlob(size_t nSrcLen, size_t& nDstSize, uint32_t& nCrc32, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const;
+
 	SigMap_Pb m_Cache;
 	bool m_bInitialized;
 };
 
+#pragma pack(push, 1)
 struct SigDBHeader_t
 {
 	int m_nMagic;
-	int m_nVersion;
+	uint16_t m_nMajorVersion;
-	FILETIME m_FileTime;
+	uint16_t m_nMinorVersion;
+	uint64_t m_nBlobSizeMem;
+	uint64_t m_nBlobSizeDisk;
+	uint32_t m_nBlobHash;
 };
+#pragma pack(pop)
 
 #endif // !SIGCACHE_H