diff --git a/r5dev/tier1/IConVar.cpp b/r5dev/tier1/IConVar.cpp index 0c94f706..65278c37 100644 --- a/r5dev/tier1/IConVar.cpp +++ b/r5dev/tier1/IConVar.cpp @@ -131,9 +131,10 @@ void ConVar::Init(void) const #endif // !DEDICATED //------------------------------------------------------------------------- // FILESYSTEM | - fs_warning_level_sdk = new ConVar("fs_warning_level_sdk" , "0", FCVAR_DEVELOPMENTONLY, "Set the SDK filesystem warning level.", false, 0.f, false, 0.f, nullptr, nullptr); - fs_show_warning_output = new ConVar("fs_show_warning_output" , "0", FCVAR_DEVELOPMENTONLY, "Logs the filesystem warnings to the console, filtered by 'fs_warning_level_native' ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr); - fs_packedstore_entryblock_stats = new ConVar("fs_packedstore_entryblock_stats", "0", FCVAR_DEVELOPMENTONLY, "If set to 1, prints the stats of each file entry in the VPK during decompression ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr); + fs_warning_level_sdk = new ConVar("fs_warning_level_sdk" , "0", FCVAR_DEVELOPMENTONLY, "Set the SDK filesystem warning level.", false, 0.f, false, 0.f, nullptr, nullptr); + fs_show_warning_output = new ConVar("fs_show_warning_output" , "0", FCVAR_DEVELOPMENTONLY, "Logs the filesystem warnings to the console, filtered by 'fs_warning_level_native' ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr); + fs_packedstore_entryblock_stats = new ConVar("fs_packedstore_entryblock_stats" , "0", FCVAR_DEVELOPMENTONLY, "If set to 1, prints the stats of each file entry in the VPK during decompression ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr); + fs_packedstore_workspace = new ConVar("fs_packedstore_workspace", "platform/vpk/", FCVAR_DEVELOPMENTONLY, "Determines the current VPK workspace.", false, 0.f, false, 0.f, nullptr, nullptr); //------------------------------------------------------------------------- // MATERIALSYSTEM | #ifndef DEDICATED diff --git a/r5dev/tier1/cvar.cpp b/r5dev/tier1/cvar.cpp index 7557817f..8a7f018b 100644 --- a/r5dev/tier1/cvar.cpp +++ b/r5dev/tier1/cvar.cpp @@ -105,6 +105,7 @@ ConVar* con_suggestion_flags_realtime = nullptr; ConVar* fs_warning_level_sdk = nullptr; ConVar* fs_show_warning_output = nullptr; ConVar* fs_packedstore_entryblock_stats = nullptr; +ConVar* fs_packedstore_workspace = nullptr; //----------------------------------------------------------------------------- // MATERIALSYSTEM | #ifndef DEDICATED diff --git a/r5dev/tier1/cvar.h b/r5dev/tier1/cvar.h index ec275978..07c01f0a 100644 --- a/r5dev/tier1/cvar.h +++ b/r5dev/tier1/cvar.h @@ -102,6 +102,7 @@ extern ConVar* con_suggestion_flags_realtime; extern ConVar* fs_warning_level_sdk; extern ConVar* fs_show_warning_output; extern ConVar* fs_packedstore_entryblock_stats; +extern ConVar* fs_packedstore_workspace; //------------------------------------------------------------------------- // MATERIALSYSTEM | #ifndef DEDICATED diff --git a/r5dev/vpklib/packedstore.cpp b/r5dev/vpklib/packedstore.cpp index c72d3961..d1edab71 100644 --- a/r5dev/vpklib/packedstore.cpp +++ b/r5dev/vpklib/packedstore.cpp @@ -20,8 +20,8 @@ void CPackedStore::InitLzCompParams(void) { /*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ m_lzCompParams.m_dict_size_log2 = RVPK_DICT_SIZE; - m_lzCompParams.m_level = lzham_compress_level::LZHAM_COMP_LEVEL_UBER; - m_lzCompParams.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING | lzham_compress_flags::LZHAM_COMP_FLAG_TRADEOFF_DECOMPRESSION_RATE_FOR_COMP_RATIO; + m_lzCompParams.m_level = lzham_compress_level::LZHAM_COMP_LEVEL_FASTER; + //m_lzCompParams.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING | lzham_compress_flags::LZHAM_COMP_FLAG_TRADEOFF_DECOMPRESSION_RATE_FOR_COMP_RATIO; m_lzCompParams.m_max_helper_threads = -1; } @@ -110,6 +110,22 @@ vector CPackedStore::GetEntryBlocks(CIOStream* pReader) return vBlocks; } +//----------------------------------------------------------------------------- +// Purpose: scans the input directory and returns the paths to the vector +//----------------------------------------------------------------------------- +vector CPackedStore::GetEntryPaths(const string& svPathIn) const +{ + vector vPaths; + for (const fs::directory_entry& dirEntry : fs::recursive_directory_iterator(fs_packedstore_workspace->GetString())) + { + if (!GetExtension(dirEntry.path().u8string()).empty()) + { + vPaths.push_back(ConvertToUnixPath(dirEntry.path().u8string())); + } + } + return vPaths; +} + //----------------------------------------------------------------------------- // Purpose: formats the entry block path //----------------------------------------------------------------------------- @@ -144,17 +160,10 @@ string CPackedStore::StripLocalePrefix(string svPackDirFile) //----------------------------------------------------------------------------- // Purpose: validates extraction result with precomputed ADLER32 hash //----------------------------------------------------------------------------- -void CPackedStore::ValidateAdler32PostDecomp(string svAssetFile) +void CPackedStore::ValidateAdler32PostDecomp(const string& svAssetFile) { - uint32_t adler_init = {}; - ifstream istream(svAssetFile, fstream::binary); - - istream.seekg(0, fstream::end); - m_vHashBuffer.resize(istream.tellg()); - istream.seekg(0, fstream::beg); - istream.read((char*)m_vHashBuffer.data(), m_vHashBuffer.size()); - - m_nAdler32 = adler32::update(adler_init, m_vHashBuffer.data(), m_vHashBuffer.size()); + CIOStream reader(svAssetFile, CIOStream::Mode_t::READ); + m_nAdler32 = adler32::update(m_nAdler32, reader.GetData(), reader.GetSize()); if (m_nAdler32 != m_nAdler32_Internal) { @@ -162,82 +171,67 @@ void CPackedStore::ValidateAdler32PostDecomp(string svAssetFile) m_nAdler32 = 0; m_nAdler32_Internal = 0; } - - istream.close(); - m_vHashBuffer.clear(); } //----------------------------------------------------------------------------- // Purpose: validates extraction result with precomputed CRC32 hash //----------------------------------------------------------------------------- -void CPackedStore::ValidateCRC32PostDecomp(string svDirAsset) +void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetFile) { - uint32_t crc32_init = {}; - ifstream istream(svDirAsset, fstream::binary); - - istream.seekg(0, fstream::end); - m_vHashBuffer.resize(istream.tellg()); - istream.seekg(0, fstream::beg); - istream.read((char*)m_vHashBuffer.data(), m_vHashBuffer.size()); - - m_nCrc32 = crc32::update(crc32_init, m_vHashBuffer.data(), m_vHashBuffer.size()); + CIOStream reader(svAssetFile, CIOStream::Mode_t::READ); + m_nCrc32 = crc32::update(m_nCrc32, reader.GetData(), reader.GetSize()); if (m_nCrc32 != m_nCrc32_Internal) { - Warning(eDLL_T::FS, "Warning: CRC32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", svDirAsset.c_str(), m_nCrc32, m_nCrc32_Internal); + Warning(eDLL_T::FS, "Warning: CRC32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", svAssetFile.c_str(), m_nCrc32, m_nCrc32_Internal); m_nCrc32 = 0; m_nCrc32_Internal = 0; } - - istream.close(); - m_vHashBuffer.clear(); } void CPackedStore::PackAll(string svDirIn, string svPathOut) { - vector uData; // Raw file data + CIOStream writer("client_mp_rr_arena_divinity.bsp.pak000_000.vpk", CIOStream::Mode_t::WRITE); + vector vEntryBlocks; + vector vPaths = GetEntryPaths(svDirIn); - ifstream iData(svDirIn, fstream::binary); - - iData.seekg(0, fstream::end); - uData.resize(iData.tellg()); - iData.seekg(0, fstream::beg); - iData.read(reinterpret_cast(uData.data()), uData.size()); - - ofstream oBlock(svDirIn, fstream::binary); - vEntryBlocks.push_back(VPKEntryBlock_t(uData, oBlock.tellp(), 0, 0x101, 0, svDirIn)); - - for (size_t i = 0; i < vEntryBlocks.size(); i++) + for (size_t i = 0; i < vPaths.size(); i++) { - for (VPKEntryDescriptor_t& entry : vEntryBlocks.at(i).m_vvEntries) + CIOStream reader(vPaths[i], CIOStream::Mode_t::READ); + if (reader.IsReadable()) { - uint8_t* pSrc = new uint8_t[entry.m_nUncompressedSize]; - uint8_t* pDest = new uint8_t[entry.m_nUncompressedSize]; - - iData.seekg(entry.m_nArchiveOffset); // Seek to entry offset in archive. - iData.read(reinterpret_cast(pSrc), entry.m_nUncompressedSize); // Read compressed data from archive. - - m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, pDest, &entry.m_nCompressedSize, pSrc, entry.m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal); - - if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS) + vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), 0, 0x101, 0, StringReplaceC(vPaths[i], fs_packedstore_workspace->GetString(), ""))); + for (size_t j = 0; j < vEntryBlocks[i].m_vvEntries.size(); j++) { - Error(eDLL_T::FS, "Error: failed compression for an entry within block '%s' for archive '%d'\n", vEntryBlocks.at(i).m_svBlockPath.c_str(), i); - Error(eDLL_T::FS, "'lzham::lzham_lib_compress_memory' returned with status '%d'.\n", m_lzCompStatus); + uint8_t* pSrc = new uint8_t[vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize]; + uint8_t* pDest = new uint8_t[DECOMP_MAX_BUF]; + + reader.Read(*pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize); + vEntryBlocks[i].m_vvEntries[j].m_nArchiveOffset = writer.GetPosition(); + + m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, pDest, &vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize, pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal); + if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS) + { + Error(eDLL_T::FS, "Error: failed compression for an entry within block '%s' for archive '%d'\n", vEntryBlocks.at(i).m_svBlockPath.c_str(), vEntryBlocks.at(i).m_iArchiveIndex); + Error(eDLL_T::FS, "'lzham::lzham_lib_compress_memory' returned with status '%d' (file will be packed without compression.)\n", m_lzCompStatus); + + vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize = vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize; + writer.Write(pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize); + } + else + writer.Write(pDest, vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize); + + vEntryBlocks[i].m_vvEntries[j].m_bIsCompressed = vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize != vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize; + + delete[] pSrc; + delete[] pDest; } - - entry.m_nArchiveOffset = oBlock.tellp(); - entry.m_bIsCompressed = entry.m_nCompressedSize != entry.m_nUncompressedSize; - - oBlock.write(reinterpret_cast(pDest), entry.m_nCompressedSize); - - delete[] pSrc; - delete[] pDest; } } VPKDir_t vDir = VPKDir_t(); - vDir.Build("englishclient_mp_rr_test.bsp.pak000_dir.vpk", vEntryBlocks); // [!!! <> !!!] + vDir.Build("englishclient_mp_rr_arena_divinity.bsp.pak000_dir.vpk", vEntryBlocks); // [!!! <> !!!] } //----------------------------------------------------------------------------- @@ -355,24 +349,22 @@ VPKEntryBlock_t::VPKEntryBlock_t(CIOStream* reader, string svPath) this->m_vvEntries.push_back(entry); } while (reader->Read() != 0xFFFF); } - +#undef min VPKEntryBlock_t::VPKEntryBlock_t(const vector &vData, int64_t nOffset, uint16_t nArchiveIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, string svBlockPath) { - m_nCrc32 = crc32::update(NULL, vData.data(), vData.size()); + m_nCrc32 = crc32::update(m_nCrc32, vData.data(), vData.size()); m_nPreloadBytes = 0; m_iArchiveIndex = nArchiveIndex; m_svBlockPath = svBlockPath; int nEntryCount = (vData.size() + RVPK_MAX_BLOCK - 1) / RVPK_MAX_BLOCK; - m_vvEntries = vector(nEntryCount); - - int64_t nDataSize = vData.size(); + uint64_t nDataSize = vData.size(); int64_t nCurrentOffset = nOffset; for (int i = 0; i < nEntryCount; i++) { - int64_t nSize = min(RVPK_MAX_BLOCK, nDataSize); + uint64_t nSize = std::min(RVPK_MAX_BLOCK, nDataSize); nDataSize -= nSize; - m_vvEntries[i] = VPKEntryDescriptor_t(nEntryFlags, nTextureFlags, nCurrentOffset, nSize, nSize); + m_vvEntries.push_back(VPKEntryDescriptor_t(nEntryFlags, nTextureFlags, nCurrentOffset, nSize, nSize)); nCurrentOffset += nSize; } } @@ -406,9 +398,7 @@ VPKEntryDescriptor_t::VPKEntryDescriptor_t(CIOStream* pReader) //----------------------------------------------------------------------------- VPKDir_t::VPKDir_t(const string& svPath) { - CIOStream reader; - - reader.Open(svPath, eStreamFileMode::READ); + CIOStream reader(svPath, CIOStream::Mode_t::READ); reader.Read(this->m_vHeader.m_nFileMagic); if (this->m_vHeader.m_nFileMagic != RVPK_DIR_MAGIC) @@ -459,7 +449,7 @@ VPKDir_t::VPKDir_t(const string& svPath) //----------------------------------------------------------------------------- void VPKDir_t::Build(const string& svFileName, const vector& vEntryBlocks) { - CIOStream writer(svFileName, eStreamFileMode::WRITE); + CIOStream writer(svFileName, CIOStream::Mode_t::WRITE); auto vMap = std::map>>(); writer.Write(this->m_vHeader.m_nFileMagic); @@ -471,9 +461,13 @@ void VPKDir_t::Build(const string& svFileName, const vector& vE for (VPKEntryBlock_t vBlock : vEntryBlocks) { string svExtension = GetExtension(vBlock.m_svBlockPath); - string svFileName = GetFileName(vBlock.m_svBlockPath); + string svFileName = GetFileName(vBlock.m_svBlockPath, true); string svFilePath = RemoveFileName(vBlock.m_svBlockPath); + if (svFilePath.empty()) + { + svFilePath = " "; // Has to be padded with a space character if empty. + } if (!vMap.count(svExtension)) { vMap.insert({ svExtension, std::map>() }); @@ -521,12 +515,12 @@ void VPKDir_t::Build(const string& svFileName, const vector& vE } } } - writer.Write('\0'); } writer.Write('\0'); } writer.Write('\0'); } + writer.Write('\0'); m_vHeader.m_nTreeSize = static_cast(writer.GetPosition() - sizeof(VPKHeader_t)); writer.SetPosition(offsetof(VPKDir_t, m_vHeader.m_nTreeSize)); diff --git a/r5dev/vpklib/packedstore.h b/r5dev/vpklib/packedstore.h index 6d9db2c4..862c006d 100644 --- a/r5dev/vpklib/packedstore.h +++ b/r5dev/vpklib/packedstore.h @@ -88,7 +88,6 @@ struct VPKDir_t class CPackedStore { - vector m_vHashBuffer {}; // Buffer for post decomp file validation. size_t m_nEntryCount {}; // Entry per-block incrementor. lzham_uint32 m_nAdler32_Internal{}; // Internal operation Adler32 file checksum. lzham_uint32 m_nAdler32 {}; // Pre/post operation Adler32 file checksum. @@ -105,12 +104,13 @@ public: VPKDir_t GetPackDirFile(string svPackDirFile); string GetPackChunkFile(string svPackDirFile, int iArchiveIndex); vector GetEntryBlocks(CIOStream* reader); + vector GetEntryPaths(const string& svPathIn) const; string FormatBlockPath(string svName, string svPath, string svExtension); string StripLocalePrefix(string svPackDirFile); void PackAll(string svDirIn, string svPathOut = ""); void UnpackAll(VPKDir_t vpk, string svPathOut = ""); - void ValidateAdler32PostDecomp(string svDirAsset); - void ValidateCRC32PostDecomp(string svDirAsset); + void ValidateAdler32PostDecomp(const string& svDirAsset); + void ValidateCRC32PostDecomp(const string& svDirAsset); }; /////////////////////////////////////////////////////////////////////////////// extern CPackedStore* g_pPackedStore;