R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add convar for adjusting LZHAM compressor max helper threads

Browse Source
This commit is contained in:
Kawe Mazidjatari 2022-11-17 20:37:12 +01:00
parent 90d87897f8
commit 8c7b58b9df
3 changed files with 27 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
r5dev/tier1/IConVar.cpp
View File

@ -184,11 +184,12 @@ void ConVar::Init(void) const
#endif // !DEDICATED
//-------------------------------------------------------------------------
// FILESYSTEM |
fs_warning_level_sdk = ConVar::Create("fs_warning_level_sdk" , "0", FCVAR_DEVELOPMENTONLY, "Set the SDK FileSystem warning level.", false, 0.f, false, 0.f, nullptr, nullptr);
fs_show_warning_output = ConVar::Create("fs_show_warning_output" , "0", FCVAR_DEVELOPMENTONLY, "Logs the FileSystem warnings to the console, filtered by 'fs_warning_level_native' ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr);
fs_packedstore_entryblock_stats = ConVar::Create("fs_packedstore_entryblock_stats" , "0", FCVAR_DEVELOPMENTONLY, "Logs the stats of each file entry in the VPK during decompression ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr);
fs_packedstore_workspace = ConVar::Create("fs_packedstore_workspace" , "platform/", FCVAR_DEVELOPMENTONLY, "Determines the current VPK workspace.", false, 0.f, false, 0.f, nullptr, nullptr);
fs_packedstore_compression_level = ConVar::Create("fs_packedstore_compression_level", "default", FCVAR_DEVELOPMENTONLY, "Determines the VPK compression level.", false, 0.f, false, 0.f, nullptr, "fastest faster default better uber");
fs_warning_level_sdk = ConVar::Create("fs_warning_level_sdk" , "0", FCVAR_DEVELOPMENTONLY, "Set the SDK FileSystem warning level.", false, 0.f, false, 0.f, nullptr, nullptr);
fs_show_warning_output = ConVar::Create("fs_show_warning_output" , "0", FCVAR_DEVELOPMENTONLY, "Logs the FileSystem warnings to the console, filtered by 'fs_warning_level_native' ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr);
fs_packedstore_entryblock_stats = ConVar::Create("fs_packedstore_entryblock_stats" , "0", FCVAR_DEVELOPMENTONLY, "Logs the stats of each file entry in the VPK during decompression ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr);
fs_packedstore_workspace = ConVar::Create("fs_packedstore_workspace" , "platform/", FCVAR_DEVELOPMENTONLY, "Determines the current VPK workspace.", false, 0.f, false, 0.f, nullptr, nullptr);
fs_packedstore_compression_level = ConVar::Create("fs_packedstore_compression_level", "default", FCVAR_DEVELOPMENTONLY, "Determines the VPK compression level.", false, 0.f, false, 0.f, nullptr, "fastest faster default better uber");
fs_packedstore_max_helper_threads = ConVar::Create("fs_packedstore_max_helper_threads" , "-1", FCVAR_DEVELOPMENTONLY, "Max # of additional \"helper\" threads to create during compression.", true, -1, true, LZHAM_MAX_HELPER_THREADS, nullptr, "Must range between [-1,LZHAM_MAX_HELPER_THREADS], where -1=max practical.");
//-------------------------------------------------------------------------
// MATERIALSYSTEM |
#ifndef DEDICATED

37
r5dev/vpklib/packedstore.cpp
View File

@ -21,7 +21,7 @@
#include "vpklib/packedstore.h"
//-----------------------------------------------------------------------------
// Static buffers for chunking/compressing the source files and decompressing
// Static buffers for fragmenting/compressing the source files and decompressing
//-----------------------------------------------------------------------------
static uint8_t s_EntryBuf[ENTRY_MAX_LEN];
static uint8_t s_DecompBuf[ENTRY_MAX_LEN];
@ -35,7 +35,7 @@ void CPackedStore::InitLzCompParams(void)
m_lzCompParams.m_dict_size_log2 = VPK_DICT_SIZE;
m_lzCompParams.m_level = GetCompressionLevel();
m_lzCompParams.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING;
m_lzCompParams.m_max_helper_threads = -1;
m_lzCompParams.m_max_helper_threads = fs_packedstore_max_helper_threads->GetInt();
}
//-----------------------------------------------------------------------------
@ -58,13 +58,17 @@ void CPackedStore::InitLzDecompParams(void)
VPKDir_t CPackedStore::GetDirectoryFile(const string& svPackDirFile, bool bSanitizeName) const
{
if (!bSanitizeName)
{
return VPKDir_t(svPackDirFile);
}
std::smatch smRegexMatches;
std::regex_search(svPackDirFile, smRegexMatches, BLOCK_REGEX);
if (smRegexMatches.empty())
{
return VPKDir_t(svPackDirFile);
}
string svSanitizedName = svPackDirFile;
StringReplace(svSanitizedName, smRegexMatches[0], "pak000_dir");
@ -198,7 +202,7 @@ vector<string> CPackedStore::GetEntryPaths(const string& svPathIn) const
// Purpose: scans the input directory and returns the paths to the vector if path exists in manifest
// Input : &svPathIn -
// &jManifest -
// Output : a string vector of all included entry paths
// Output : a string vector of all included and existing entry paths
//-----------------------------------------------------------------------------
vector<string> CPackedStore::GetEntryPaths(const string& svPathIn, const nlohmann::json& jManifest) const
{
@ -482,8 +486,8 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspa
vPaths = GetEntryPaths(svWorkspace);
}
uint64_t nSharedTotal = 0;
uint32_t nSharedCount = 0;
uint64_t nSharedTotal = NULL;
uint32_t nSharedCount = NULL;
for (size_t i = 0, ps = vPaths.size(); i < ps; i++)
{
@ -492,11 +496,11 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspa
if (reader.IsReadable())
{
const string svDestPath = StringReplaceC(svPath, svWorkspace, "");
uint16_t iPreloadSize = 0;
uint32_t nLoadFlags = static_cast<uint32_t>(EPackedLoadFlags::LOAD_VISIBLE) | static_cast<uint32_t>(EPackedLoadFlags::LOAD_CACHE);
uint16_t nTextureFlags = static_cast<uint16_t>(EPackedTextureFlags::TEXTURE_DEFAULT);
bool bUseCompression = true;
bool bUseDataSharing = true;
uint16_t iPreloadSize = NULL;
uint32_t nLoadFlags = static_cast<uint32_t>(EPackedLoadFlags::LOAD_VISIBLE) | static_cast<uint32_t>(EPackedLoadFlags::LOAD_CACHE);
uint16_t nTextureFlags = static_cast<uint16_t>(EPackedTextureFlags::TEXTURE_DEFAULT);
bool bUseCompression = true;
bool bUseDataSharing = true;
if (!jManifest.is_null())
{
@ -548,8 +552,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspa
if (bUseCompression)
{
m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, s_EntryBuf,
&vDescriptor.m_nCompressedSize, s_EntryBuf,
m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, s_EntryBuf, &vDescriptor.m_nCompressedSize, s_EntryBuf,
vDescriptor.m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal);
if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
@ -657,7 +660,7 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vDir, const string& svWorkspa
if (m_nChunkCount == vBlock.m_vFragments.size()) // Only validate after last entry in block had been written.
{
m_nChunkCount = 0;
m_nChunkCount = NULL;
m_nCrc32_Internal = vBlock.m_nFileCRC;
oStream.Flush();
@ -797,7 +800,7 @@ void VPKDir_t::Build(const string& svDirectoryFile, const vector<VPKEntryBlock_t
{
CIOStream writer(svDirectoryFile, CIOStream::Mode_t::WRITE);
auto vMap = std::map<string, std::map<string, std::list<VPKEntryBlock_t>>>();
uint64_t nDescriptors = 0;
uint64_t nDescriptors = NULL;
writer.Write<uint32_t>(m_vHeader.m_nHeaderMarker);
writer.Write<uint16_t>(m_vHeader.m_nMajorVersion);
@ -852,13 +855,11 @@ void VPKDir_t::Build(const string& svDirectoryFile, const vector<VPKEntryBlock_t
if (i != (nc - 1))
{
const ushort s = 0;
writer.Write(s);
writer.Write<uint16_t>(NULL);
}
else // Mark end of entry.
{
const ushort s = PACKFILEINDEX_END;
writer.Write(s);
writer.Write<uint16_t>(PACKFILEINDEX_END);
}
nDescriptors++;
}

3
r5dev/vpklib/packedstore.h
View File

@ -16,6 +16,7 @@ constexpr unsigned int VPK_MAJOR_VERSION = 2;
constexpr unsigned int VPK_MINOR_VERSION = 3;
constexpr unsigned int VPK_DICT_SIZE = 20;
constexpr int ENTRY_MAX_LEN = 1024 * 1024;
constexpr int PACKFILEPATCH_MAX = 512;
constexpr int PACKFILEINDEX_END = 0xffff;
static const std::regex BLOCK_REGEX{ R"(pak000_([0-9]{3}))" };
@ -180,7 +181,7 @@ public:
void ValidateCRC32PostDecomp(const string& svDirAsset);
private:
size_t m_nChunkCount; // The number of patches (multi-pack file).
size_t m_nChunkCount; // The number of fragments for this asset.
lzham_uint32 m_nAdler32_Internal; // Internal operation Adler32 file checksum.
lzham_uint32 m_nAdler32; // Pre/post operation Adler32 file checksum.
lzham_uint32 m_nCrc32_Internal; // Internal operation Crc32 file checksum.