From ec7582e476f30add707475d4c2b39fcd6ab17bf7 Mon Sep 17 00:00:00 2001
From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com>
Date: Thu, 2 Jun 2022 01:57:18 +0200
Subject: [PATCH] Fixed VPK repack with multiple files
VPK's could now be repacked with compression and multiple files. The output is identical to the input as far as currently tested (mp_rr_canyonlands_staging).
---
r5dev/tier1/IConVar.cpp | 7 +-
r5dev/tier1/cvar.cpp | 1 +
r5dev/tier1/cvar.h | 1 +
r5dev/vpklib/packedstore.cpp | 144 +++++++++++++++++------------------
r5dev/vpklib/packedstore.h | 6 +-
5 files changed, 78 insertions(+), 81 deletions(-)
diff --git a/r5dev/tier1/IConVar.cpp b/r5dev/tier1/IConVar.cpp
index 0c94f706..65278c37 100644
--- a/r5dev/tier1/IConVar.cpp
+++ b/r5dev/tier1/IConVar.cpp
@@ -131,9 +131,10 @@ void ConVar::Init(void) const
#endif // !DEDICATED
//-------------------------------------------------------------------------
// FILESYSTEM |
- fs_warning_level_sdk = new ConVar("fs_warning_level_sdk" , "0", FCVAR_DEVELOPMENTONLY, "Set the SDK filesystem warning level.", false, 0.f, false, 0.f, nullptr, nullptr);
- fs_show_warning_output = new ConVar("fs_show_warning_output" , "0", FCVAR_DEVELOPMENTONLY, "Logs the filesystem warnings to the console, filtered by 'fs_warning_level_native' ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr);
- fs_packedstore_entryblock_stats = new ConVar("fs_packedstore_entryblock_stats", "0", FCVAR_DEVELOPMENTONLY, "If set to 1, prints the stats of each file entry in the VPK during decompression ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr);
+ fs_warning_level_sdk = new ConVar("fs_warning_level_sdk" , "0", FCVAR_DEVELOPMENTONLY, "Set the SDK filesystem warning level.", false, 0.f, false, 0.f, nullptr, nullptr);
+ fs_show_warning_output = new ConVar("fs_show_warning_output" , "0", FCVAR_DEVELOPMENTONLY, "Logs the filesystem warnings to the console, filtered by 'fs_warning_level_native' ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr);
+ fs_packedstore_entryblock_stats = new ConVar("fs_packedstore_entryblock_stats" , "0", FCVAR_DEVELOPMENTONLY, "If set to 1, prints the stats of each file entry in the VPK during decompression ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr);
+ fs_packedstore_workspace = new ConVar("fs_packedstore_workspace", "platform/vpk/", FCVAR_DEVELOPMENTONLY, "Determines the current VPK workspace.", false, 0.f, false, 0.f, nullptr, nullptr);
//-------------------------------------------------------------------------
// MATERIALSYSTEM |
#ifndef DEDICATED
diff --git a/r5dev/tier1/cvar.cpp b/r5dev/tier1/cvar.cpp
index 7557817f..8a7f018b 100644
--- a/r5dev/tier1/cvar.cpp
+++ b/r5dev/tier1/cvar.cpp
@@ -105,6 +105,7 @@ ConVar* con_suggestion_flags_realtime = nullptr;
ConVar* fs_warning_level_sdk = nullptr;
ConVar* fs_show_warning_output = nullptr;
ConVar* fs_packedstore_entryblock_stats = nullptr;
+ConVar* fs_packedstore_workspace = nullptr;
//-----------------------------------------------------------------------------
// MATERIALSYSTEM |
#ifndef DEDICATED
diff --git a/r5dev/tier1/cvar.h b/r5dev/tier1/cvar.h
index ec275978..07c01f0a 100644
--- a/r5dev/tier1/cvar.h
+++ b/r5dev/tier1/cvar.h
@@ -102,6 +102,7 @@ extern ConVar* con_suggestion_flags_realtime;
extern ConVar* fs_warning_level_sdk;
extern ConVar* fs_show_warning_output;
extern ConVar* fs_packedstore_entryblock_stats;
+extern ConVar* fs_packedstore_workspace;
//-------------------------------------------------------------------------
// MATERIALSYSTEM |
#ifndef DEDICATED
diff --git a/r5dev/vpklib/packedstore.cpp b/r5dev/vpklib/packedstore.cpp
index c72d3961..d1edab71 100644
--- a/r5dev/vpklib/packedstore.cpp
+++ b/r5dev/vpklib/packedstore.cpp
@@ -20,8 +20,8 @@ void CPackedStore::InitLzCompParams(void)
{
/*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/
m_lzCompParams.m_dict_size_log2 = RVPK_DICT_SIZE;
- m_lzCompParams.m_level = lzham_compress_level::LZHAM_COMP_LEVEL_UBER;
- m_lzCompParams.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING | lzham_compress_flags::LZHAM_COMP_FLAG_TRADEOFF_DECOMPRESSION_RATE_FOR_COMP_RATIO;
+ m_lzCompParams.m_level = lzham_compress_level::LZHAM_COMP_LEVEL_FASTER;
+ //m_lzCompParams.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING | lzham_compress_flags::LZHAM_COMP_FLAG_TRADEOFF_DECOMPRESSION_RATE_FOR_COMP_RATIO;
m_lzCompParams.m_max_helper_threads = -1;
}
@@ -110,6 +110,22 @@ vector<VPKEntryBlock_t> CPackedStore::GetEntryBlocks(CIOStream* pReader)
return vBlocks;
}
+//-----------------------------------------------------------------------------
+// Purpose: scans the input directory and returns the paths to the vector
+//-----------------------------------------------------------------------------
+vector<string> CPackedStore::GetEntryPaths(const string& svPathIn) const
+{
+ vector<string> vPaths;
+ for (const fs::directory_entry& dirEntry : fs::recursive_directory_iterator(fs_packedstore_workspace->GetString()))
+ {
+ if (!GetExtension(dirEntry.path().u8string()).empty())
+ {
+ vPaths.push_back(ConvertToUnixPath(dirEntry.path().u8string()));
+ }
+ }
+ return vPaths;
+}
+
//-----------------------------------------------------------------------------
// Purpose: formats the entry block path
//-----------------------------------------------------------------------------
@@ -144,17 +160,10 @@ string CPackedStore::StripLocalePrefix(string svPackDirFile)
//-----------------------------------------------------------------------------
// Purpose: validates extraction result with precomputed ADLER32 hash
//-----------------------------------------------------------------------------
-void CPackedStore::ValidateAdler32PostDecomp(string svAssetFile)
+void CPackedStore::ValidateAdler32PostDecomp(const string& svAssetFile)
{
- uint32_t adler_init = {};
- ifstream istream(svAssetFile, fstream::binary);
-
- istream.seekg(0, fstream::end);
- m_vHashBuffer.resize(istream.tellg());
- istream.seekg(0, fstream::beg);
- istream.read((char*)m_vHashBuffer.data(), m_vHashBuffer.size());
-
- m_nAdler32 = adler32::update(adler_init, m_vHashBuffer.data(), m_vHashBuffer.size());
+ CIOStream reader(svAssetFile, CIOStream::Mode_t::READ);
+ m_nAdler32 = adler32::update(m_nAdler32, reader.GetData(), reader.GetSize());
if (m_nAdler32 != m_nAdler32_Internal)
{
@@ -162,82 +171,67 @@ void CPackedStore::ValidateAdler32PostDecomp(string svAssetFile)
m_nAdler32 = 0;
m_nAdler32_Internal = 0;
}
-
- istream.close();
- m_vHashBuffer.clear();
}
//-----------------------------------------------------------------------------
// Purpose: validates extraction result with precomputed CRC32 hash
//-----------------------------------------------------------------------------
-void CPackedStore::ValidateCRC32PostDecomp(string svDirAsset)
+void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetFile)
{
- uint32_t crc32_init = {};
- ifstream istream(svDirAsset, fstream::binary);
-
- istream.seekg(0, fstream::end);
- m_vHashBuffer.resize(istream.tellg());
- istream.seekg(0, fstream::beg);
- istream.read((char*)m_vHashBuffer.data(), m_vHashBuffer.size());
-
- m_nCrc32 = crc32::update(crc32_init, m_vHashBuffer.data(), m_vHashBuffer.size());
+ CIOStream reader(svAssetFile, CIOStream::Mode_t::READ);
+ m_nCrc32 = crc32::update(m_nCrc32, reader.GetData(), reader.GetSize());
if (m_nCrc32 != m_nCrc32_Internal)
{
- Warning(eDLL_T::FS, "Warning: CRC32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", svDirAsset.c_str(), m_nCrc32, m_nCrc32_Internal);
+ Warning(eDLL_T::FS, "Warning: CRC32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", svAssetFile.c_str(), m_nCrc32, m_nCrc32_Internal);
m_nCrc32 = 0;
m_nCrc32_Internal = 0;
}
-
- istream.close();
- m_vHashBuffer.clear();
}
void CPackedStore::PackAll(string svDirIn, string svPathOut)
{
- vector<uint8_t> uData; // Raw file data
+ CIOStream writer("client_mp_rr_arena_divinity.bsp.pak000_000.vpk", CIOStream::Mode_t::WRITE);
+
vector<VPKEntryBlock_t> vEntryBlocks;
+ vector<string> vPaths = GetEntryPaths(svDirIn);
- ifstream iData(svDirIn, fstream::binary);
-
- iData.seekg(0, fstream::end);
- uData.resize(iData.tellg());
- iData.seekg(0, fstream::beg);
- iData.read(reinterpret_cast<char*>(uData.data()), uData.size());
-
- ofstream oBlock(svDirIn, fstream::binary);
- vEntryBlocks.push_back(VPKEntryBlock_t(uData, oBlock.tellp(), 0, 0x101, 0, svDirIn));
-
- for (size_t i = 0; i < vEntryBlocks.size(); i++)
+ for (size_t i = 0; i < vPaths.size(); i++)
{
- for (VPKEntryDescriptor_t& entry : vEntryBlocks.at(i).m_vvEntries)
+ CIOStream reader(vPaths[i], CIOStream::Mode_t::READ);
+ if (reader.IsReadable())
{
- uint8_t* pSrc = new uint8_t[entry.m_nUncompressedSize];
- uint8_t* pDest = new uint8_t[entry.m_nUncompressedSize];
-
- iData.seekg(entry.m_nArchiveOffset); // Seek to entry offset in archive.
- iData.read(reinterpret_cast<char*>(pSrc), entry.m_nUncompressedSize); // Read compressed data from archive.
-
- m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, pDest, &entry.m_nCompressedSize, pSrc, entry.m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal);
-
- if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
+ vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), 0, 0x101, 0, StringReplaceC(vPaths[i], fs_packedstore_workspace->GetString(), "")));
+ for (size_t j = 0; j < vEntryBlocks[i].m_vvEntries.size(); j++)
{
- Error(eDLL_T::FS, "Error: failed compression for an entry within block '%s' for archive '%d'\n", vEntryBlocks.at(i).m_svBlockPath.c_str(), i);
- Error(eDLL_T::FS, "'lzham::lzham_lib_compress_memory' returned with status '%d'.\n", m_lzCompStatus);
+ uint8_t* pSrc = new uint8_t[vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize];
+ uint8_t* pDest = new uint8_t[DECOMP_MAX_BUF];
+
+ reader.Read(*pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize);
+ vEntryBlocks[i].m_vvEntries[j].m_nArchiveOffset = writer.GetPosition();
+
+ m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, pDest, &vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize, pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal);
+ if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
+ {
+ Error(eDLL_T::FS, "Error: failed compression for an entry within block '%s' for archive '%d'\n", vEntryBlocks.at(i).m_svBlockPath.c_str(), vEntryBlocks.at(i).m_iArchiveIndex);
+ Error(eDLL_T::FS, "'lzham::lzham_lib_compress_memory' returned with status '%d' (file will be packed without compression.)\n", m_lzCompStatus);
+
+ vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize = vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize;
+ writer.Write(pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize);
+ }
+ else
+ writer.Write(pDest, vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize);
+
+ vEntryBlocks[i].m_vvEntries[j].m_bIsCompressed = vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize != vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize;
+
+ delete[] pSrc;
+ delete[] pDest;
}
-
- entry.m_nArchiveOffset = oBlock.tellp();
- entry.m_bIsCompressed = entry.m_nCompressedSize != entry.m_nUncompressedSize;
-
- oBlock.write(reinterpret_cast<char*>(pDest), entry.m_nCompressedSize);
-
- delete[] pSrc;
- delete[] pDest;
}
}
VPKDir_t vDir = VPKDir_t();
- vDir.Build("englishclient_mp_rr_test.bsp.pak000_dir.vpk", vEntryBlocks); // [!!! <<DEVELOPMENT>> !!!]
+ vDir.Build("englishclient_mp_rr_arena_divinity.bsp.pak000_dir.vpk", vEntryBlocks); // [!!! <<DEVELOPMENT>> !!!]
}
//-----------------------------------------------------------------------------
@@ -355,24 +349,22 @@ VPKEntryBlock_t::VPKEntryBlock_t(CIOStream* reader, string svPath)
this->m_vvEntries.push_back(entry);
} while (reader->Read<uint16_t>() != 0xFFFF);
}
-
+#undef min
VPKEntryBlock_t::VPKEntryBlock_t(const vector<uint8_t> &vData, int64_t nOffset, uint16_t nArchiveIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, string svBlockPath)
{
- m_nCrc32 = crc32::update(NULL, vData.data(), vData.size());
+ m_nCrc32 = crc32::update(m_nCrc32, vData.data(), vData.size());
m_nPreloadBytes = 0;
m_iArchiveIndex = nArchiveIndex;
m_svBlockPath = svBlockPath;
int nEntryCount = (vData.size() + RVPK_MAX_BLOCK - 1) / RVPK_MAX_BLOCK;
- m_vvEntries = vector<VPKEntryDescriptor_t>(nEntryCount);
-
- int64_t nDataSize = vData.size();
+ uint64_t nDataSize = vData.size();
int64_t nCurrentOffset = nOffset;
for (int i = 0; i < nEntryCount; i++)
{
- int64_t nSize = min(RVPK_MAX_BLOCK, nDataSize);
+ uint64_t nSize = std::min<uint64_t>(RVPK_MAX_BLOCK, nDataSize);
nDataSize -= nSize;
- m_vvEntries[i] = VPKEntryDescriptor_t(nEntryFlags, nTextureFlags, nCurrentOffset, nSize, nSize);
+ m_vvEntries.push_back(VPKEntryDescriptor_t(nEntryFlags, nTextureFlags, nCurrentOffset, nSize, nSize));
nCurrentOffset += nSize;
}
}
@@ -406,9 +398,7 @@ VPKEntryDescriptor_t::VPKEntryDescriptor_t(CIOStream* pReader)
//-----------------------------------------------------------------------------
VPKDir_t::VPKDir_t(const string& svPath)
{
- CIOStream reader;
-
- reader.Open(svPath, eStreamFileMode::READ);
+ CIOStream reader(svPath, CIOStream::Mode_t::READ);
reader.Read<uint32_t>(this->m_vHeader.m_nFileMagic);
if (this->m_vHeader.m_nFileMagic != RVPK_DIR_MAGIC)
@@ -459,7 +449,7 @@ VPKDir_t::VPKDir_t(const string& svPath)
//-----------------------------------------------------------------------------
void VPKDir_t::Build(const string& svFileName, const vector<VPKEntryBlock_t>& vEntryBlocks)
{
- CIOStream writer(svFileName, eStreamFileMode::WRITE);
+ CIOStream writer(svFileName, CIOStream::Mode_t::WRITE);
auto vMap = std::map<string, std::map<string, std::list<VPKEntryBlock_t>>>();
writer.Write<uint32_t>(this->m_vHeader.m_nFileMagic);
@@ -471,9 +461,13 @@ void VPKDir_t::Build(const string& svFileName, const vector<VPKEntryBlock_t>& vE
for (VPKEntryBlock_t vBlock : vEntryBlocks)
{
string svExtension = GetExtension(vBlock.m_svBlockPath);
- string svFileName = GetFileName(vBlock.m_svBlockPath);
+ string svFileName = GetFileName(vBlock.m_svBlockPath, true);
string svFilePath = RemoveFileName(vBlock.m_svBlockPath);
+ if (svFilePath.empty())
+ {
+ svFilePath = " "; // Has to be padded with a space character if empty.
+ }
if (!vMap.count(svExtension))
{
vMap.insert({ svExtension, std::map<string, std::list<VPKEntryBlock_t>>() });
@@ -521,12 +515,12 @@ void VPKDir_t::Build(const string& svFileName, const vector<VPKEntryBlock_t>& vE
}
}
}
- writer.Write<uint8_t>('\0');
}
writer.Write<uint8_t>('\0');
}
writer.Write<uint8_t>('\0');
}
+ writer.Write<uint8_t>('\0');
m_vHeader.m_nTreeSize = static_cast<uint32_t>(writer.GetPosition() - sizeof(VPKHeader_t));
writer.SetPosition(offsetof(VPKDir_t, m_vHeader.m_nTreeSize));
diff --git a/r5dev/vpklib/packedstore.h b/r5dev/vpklib/packedstore.h
index 6d9db2c4..862c006d 100644
--- a/r5dev/vpklib/packedstore.h
+++ b/r5dev/vpklib/packedstore.h
@@ -88,7 +88,6 @@ struct VPKDir_t
class CPackedStore
{
- vector<uint8_t> m_vHashBuffer {}; // Buffer for post decomp file validation.
size_t m_nEntryCount {}; // Entry per-block incrementor.
lzham_uint32 m_nAdler32_Internal{}; // Internal operation Adler32 file checksum.
lzham_uint32 m_nAdler32 {}; // Pre/post operation Adler32 file checksum.
@@ -105,12 +104,13 @@ public:
VPKDir_t GetPackDirFile(string svPackDirFile);
string GetPackChunkFile(string svPackDirFile, int iArchiveIndex);
vector<VPKEntryBlock_t> GetEntryBlocks(CIOStream* reader);
+ vector<string> GetEntryPaths(const string& svPathIn) const;
string FormatBlockPath(string svName, string svPath, string svExtension);
string StripLocalePrefix(string svPackDirFile);
void PackAll(string svDirIn, string svPathOut = "");
void UnpackAll(VPKDir_t vpk, string svPathOut = "");
- void ValidateAdler32PostDecomp(string svDirAsset);
- void ValidateCRC32PostDecomp(string svDirAsset);
+ void ValidateAdler32PostDecomp(const string& svDirAsset);
+ void ValidateCRC32PostDecomp(const string& svDirAsset);
};
///////////////////////////////////////////////////////////////////////////////
extern CPackedStore* g_pPackedStore;