From 22029abfc28216c944fe44ca1a4238027baf8e66 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Sun, 5 Jun 2022 02:24:44 +0200 Subject: [PATCH] Fixed compression bug for VPK system Fixed bug where the compressed buffer size equals the source buffer size getting packed into the VPK chunk. In all occurrences this resulted in corrupted data upon export. The reason for data being corrupt upon export is because the engine checks the equality of the compressed and decompressed size fields in the directory file, if they are equal, the engine doesn't attempt to decompress the block. So we end up with a still compressed block on the disk (technically not corrupt in the context of the compression lib, but useless as-is on the disk). If a compressed file doesn't get lower in size we are better out storing it rather than compressing it. Added a new condition in lzham::lzham_lib_compress_memory which checks source and destination buffer size equality. --- r5dev/thirdparty/lzham/lzhamcomp/lzham_lzcomp.cpp | 7 +++++++ r5dev/vpklib/packedstore.cpp | 15 +++++++-------- r5dev/vpklib/packedstore.h | 3 +-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/r5dev/thirdparty/lzham/lzhamcomp/lzham_lzcomp.cpp b/r5dev/thirdparty/lzham/lzhamcomp/lzham_lzcomp.cpp index 4e54c736..55a86de7 100644 --- a/r5dev/thirdparty/lzham/lzhamcomp/lzham_lzcomp.cpp +++ b/r5dev/thirdparty/lzham/lzhamcomp/lzham_lzcomp.cpp @@ -374,6 +374,13 @@ namespace lzham if (pCrc32) *pCrc32 = pCompressor->get_src_crc32(); + if (comp_data.size() == dst_buf_size) + { + lzham_delete(pTP); + lzham_delete(pCompressor); + return LZHAM_COMP_STATUS_FAILED; + } + if (comp_data.size() > dst_buf_size) { lzham_delete(pTP); diff --git a/r5dev/vpklib/packedstore.cpp b/r5dev/vpklib/packedstore.cpp index 2fbe5603..d0af8f8f 100644 --- a/r5dev/vpklib/packedstore.cpp +++ b/r5dev/vpklib/packedstore.cpp @@ -385,7 +385,6 @@ void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const uint16_t nTextureFlags = static_cast(EPackedTextureFlags::TEXTURE_DEFAULT); // !TODO: Reverse these. bool bUseCompression = true; bool bUseDataSharing = true; - string svEntryHash; if (!jManifest.is_null()) { @@ -410,8 +409,8 @@ void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), nPreloadData, 0, nEntryFlags, nTextureFlags, StringReplaceC(vPaths[i], svPathIn, ""))); for (size_t j = 0; j < vEntryBlocks[i].m_vvEntries.size(); j++) { - uint8_t* pSrc = new uint8_t[vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize]; - uint8_t* pDest = new uint8_t[COMP_MAX]; + uint8_t* pSrc = new uint8_t[vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize]; + uint8_t* pDest = new uint8_t[vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize]; bool bShared = false; bool bCompressed = bUseCompression; @@ -440,16 +439,16 @@ void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const if (bUseDataSharing) { - svEntryHash = sha1(string(reinterpret_cast(pDest), vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize)); + string svEntryHash = sha1(string(reinterpret_cast(pDest), vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize)); - if (auto it{ m_mEntryHasMap.find(svEntryHash) }; it != std::end(m_mEntryHasMap)) + if (auto it{ m_mEntryHashMap.find(svEntryHash) }; it != std::end(m_mEntryHashMap)) { vEntryBlocks[i].m_vvEntries[j] = it->second; bShared = true; } - else + else // Add entry to hashmap. { - m_mEntryHasMap.insert({ svEntryHash, vEntryBlocks[i].m_vvEntries[j] }); + m_mEntryHashMap.insert({ svEntryHash, vEntryBlocks[i].m_vvEntries[j] }); bShared = false; } } @@ -464,7 +463,7 @@ void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const } } - m_mEntryHasMap.clear(); + m_mEntryHashMap.clear(); VPKDir_t vDir = VPKDir_t(); vDir.Build(svPathOut + vPair.m_svDirectoryName, vEntryBlocks); } diff --git a/r5dev/vpklib/packedstore.h b/r5dev/vpklib/packedstore.h index 25bb9737..8d64bb21 100644 --- a/r5dev/vpklib/packedstore.h +++ b/r5dev/vpklib/packedstore.h @@ -8,7 +8,6 @@ constexpr unsigned int VPK_MINOR_VERSION = 3; constexpr unsigned int RVPK_DICT_SIZE = 20; constexpr int ENTRY_MAX = 1024 * 1024; -constexpr int COMP_MAX = 2024 * 2024; const vector DIR_CONTEXT = { "server", "client" }; const vector DIR_LOCALE = { "english", "french", "german", "italian", "japanese", "korean", "polish", "portuguese", "russian", "spanish", "tchinese" }; @@ -119,7 +118,7 @@ class CPackedStore lzham_compress_status_t m_lzCompStatus {}; // LZham compression status. lzham_decompress_params m_lzDecompParams {}; // LZham decompression parameters. lzham_decompress_status_t m_lzDecompStatus {}; // LZham decompression status. - std::unordered_map m_mEntryHasMap{}; + std::unordered_map m_mEntryHashMap{}; public: void InitLzCompParams(void);