R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use static buffer for file chunking and compressing during VPK packing

Browse Source 
Avoid using new every time. Chunk can be at most 1024 * 1024.
This commit is contained in:
Kawe Mazidjatari 2022-10-02 12:17:03 +02:00
parent 33e1795cf1
commit 0451a75f85
2 changed files with 36 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
r5dev/vpklib/packedstore.cpp
View File

@ -13,6 +13,13 @@
#include "mathlib/sha1.h" #include "mathlib/sha1.h"
#include "vpklib/packedstore.h" #include "vpklib/packedstore.h"
//-----------------------------------------------------------------------------
// Single static buffer for the source entry, which is the source file chunked
// into ENTRY_MAX_LEN bytes (1024 * 1024). The compressed chunk size is always
// lower than the source chunk size with the light modifications done to the LZHAM lib.
//-----------------------------------------------------------------------------
static uint8_t s_pEntryBuffer[ENTRY_MAX_LEN];
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: initialize parameters for compression algorithm // Purpose: initialize parameters for compression algorithm
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
@ -499,18 +506,15 @@ void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const
vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), iPreloadSize, 0, nLoadFlags, nTextureFlags, svDestPath)); vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), iPreloadSize, 0, nLoadFlags, nTextureFlags, svDestPath));
for (size_t j = 0; j < vEntryBlocks[i].m_vChunks.size(); j++) for (size_t j = 0; j < vEntryBlocks[i].m_vChunks.size(); j++)
{ {
uint8_t* pSrc = new uint8_t[vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize];
uint8_t* pDest = new uint8_t[vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize];
bool bShared = false; bool bShared = false;
bool bCompressed = bUseCompression; bool bCompressed = bUseCompression;
reader.Read(*pSrc, vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize); reader.Read(*s_pEntryBuffer, vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize);
vEntryBlocks[i].m_vChunks[j].m_nArchiveOffset = writer.GetPosition(); vEntryBlocks[i].m_vChunks[j].m_nArchiveOffset = writer.GetPosition();
if (bUseDataSharing) if (bUseDataSharing)
{ {
string svEntryHash = sha1(string(reinterpret_cast<char*>(pSrc), vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize)); string svEntryHash = sha1(string(reinterpret_cast<char*>(s_pEntryBuffer), vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize));
auto p = m_mChunkHashMap.insert({ svEntryHash, vEntryBlocks[i].m_vChunks[j] }); auto p = m_mChunkHashMap.insert({ svEntryHash, vEntryBlocks[i].m_vChunks[j] });
if (p.second) if (p.second)
@ -532,34 +536,33 @@ void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const
{ {
if (bUseCompression) if (bUseCompression)
{ {
m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, pDest, m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, s_pEntryBuffer,
&vEntryBlocks[i].m_vChunks[j].m_nCompressedSize, pSrc, &vEntryBlocks[i].m_vChunks[j].m_nCompressedSize, s_pEntryBuffer,
vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal); vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal);
if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS) if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
{ {
Warning(eDLL_T::FS, "Status '%d' for chunk '%zu' within entry '%zu' in block '%hu' (chunk packed without compression)\n", Warning(eDLL_T::FS, "Status '%d' for chunk '%zu' within entry '%zu' in block '%hu' (chunk packed without compression)\n",
m_lzCompStatus, j, i, vEntryBlocks[i].m_iPackFileIndex); m_lzCompStatus, j, i, vEntryBlocks[i].m_iPackFileIndex);
vEntryBlocks[i].m_vChunks[j].m_nCompressedSize = vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize; vEntryBlocks[i].m_vChunks[j].m_nCompressedSize = vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize;
memmove(pDest, pSrc, vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize);
} }
} }
else // Write data uncompressed. else // Write data uncompressed.
{ {
vEntryBlocks[i].m_vChunks[j].m_nCompressedSize = vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize; vEntryBlocks[i].m_vChunks[j].m_nCompressedSize = vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize;
memmove(pDest, pSrc, vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize);
}
vEntryBlocks[i].m_vChunks[j].m_bIsCompressed = vEntryBlocks[i].m_vChunks[j].m_nCompressedSize != vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize;
writer.Write(pDest, vEntryBlocks[i].m_vChunks[j].m_nCompressedSize);
} }
delete[] pDest; vEntryBlocks[i].m_vChunks[j].m_bIsCompressed = vEntryBlocks[i].m_vChunks[j].m_nCompressedSize != vEntryBlocks[i].m_vChunks[j].m_nUncompressedSize;
delete[] pSrc; writer.Write(s_pEntryBuffer, vEntryBlocks[i].m_vChunks[j].m_nCompressedSize);
}
} }
} }
} }
DevMsg(eDLL_T::FS, "*** Build block totaling '%zu' bytes with '%zu' shared bytes among '%lu' chunks\n", writer.GetPosition(), nSharedTotal, nSharedCount); DevMsg(eDLL_T::FS, "*** Build block totaling '%zu' bytes with '%zu' shared bytes among '%lu' chunks\n", writer.GetPosition(), nSharedTotal, nSharedCount);
m_mChunkHashMap.clear(); m_mChunkHashMap.clear();
memset(s_pEntryBuffer, '\0', sizeof(s_pEntryBuffer));
VPKDir_t vDir = VPKDir_t(); VPKDir_t vDir = VPKDir_t();
vDir.Build(svPathOut + vPair.m_svDirectoryName, vEntryBlocks); vDir.Build(svPathOut + vPair.m_svDirectoryName, vEntryBlocks);

21
r5dev/vpklib/packedstore.h
View File

@ -8,8 +8,25 @@ constexpr unsigned int VPK_MINOR_VERSION = 3;
constexpr unsigned int VPK_DICT_SIZE = 20; constexpr unsigned int VPK_DICT_SIZE = 20;
constexpr int ENTRY_MAX_LEN = 1024 * 1024; constexpr int ENTRY_MAX_LEN = 1024 * 1024;
const vector<string> DIR_CONTEXT = { "server", "client" }; static const vector<string> DIR_CONTEXT =
const vector<string> DIR_LOCALE = { "english", "french", "german", "italian", "japanese", "korean", "polish", "portuguese", "russian", "spanish", "tchinese" }; {
"server",
"client"
};
static const vector<string> DIR_LOCALE =
{
"english",
"french",
"german",
"italian",
"japanese",
"korean",
"polish",
"portuguese",
"russian",
"spanish",
"tchinese"
};
enum class EPackedLoadFlags : int enum class EPackedLoadFlags : int
{ {