R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

ReVPK: add option for num helper threads and compress levels

Browse Source 
Allow the user to set the # amount of helper threads and the compress level of the VPK's.
This commit is contained in:
Kawe Mazidjatari 2024-01-13 14:02:46 +01:00
parent 49b82dc61c
commit 0dd630e13a
4 changed files with 47 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/common/callback.cpp
View File

@ -650,7 +650,7 @@ void VPK_Pack_f(const CCommand& args)
CPackedStoreBuilder builder;
builder.InitLzCompParams(fs_packedstore_compression_level->GetString(), fs_packedstore_max_helper_threads->GetInt());
builder.InitLzEncoder(fs_packedstore_max_helper_threads->GetInt(), fs_packedstore_compression_level->GetString());
builder.PackWorkspace(pair, fs_packedstore_workspace->GetString(), "vpk/");
timer.End();
@ -682,7 +682,7 @@ void VPK_Unpack_f(const CCommand& args)
CPackedStoreBuilder builder;
builder.InitLzDecompParams();
builder.InitLzDecoder();
builder.UnpackWorkspace(vpk, fs_packedstore_workspace->GetString());
timer.End();

35
src/revpk/revpk.cpp
View File

@ -73,20 +73,28 @@ static void ReVPK_Usage()
"ReVPK instructions and options:\n"
"For packing; run 'revpk %s' with the following parameters:\n"
"\t<%s>\t- locale prefix for the directory tree file\n"
"\t<%s>\t- whether to build for 'server' or 'client'\n"
"\t<%s>\t- the level name for the VPK files\n"
"\t<%s>\t- ( optional ) path to the workspace containing the manifest file\n"
"\t<%s>\t- ( optional ) path in which the VPK files will be built\n\n"
"\t<%s>\t- context scope for the VPK files [\"server\", \"client\"]\n"
"\t<%s>\t- level name for the VPK files\n"
"\t<%s>\t- ( optional ) path to the workspace containing the control file\n"
"\t<%s>\t- ( optional ) path in which the VPK files will be built\n"
"\t<%s>\t- ( optional ) max LZHAM helper threads [\"%d\", \"%d\"] \"%d\" ( default ) for max practical\n"
"\t<%s>\t- ( optional ) the level of compression [\"%s\", \"%s\", \"%s\", \"%s\", \"%s\"]\n\n"
"For unpacking; run 'revpk %s' with the following parameters:\n"
"\t<%s>\t- path to the directory tree file\n"
"\t<%s>\t- ( optional ) path to the target directory tree file\n"
"\t<%s>\t- name of the target directory tree file\n"
"\t<%s>\t- ( optional ) path to directory containing the target directory tree file\n"
"\t<%s>\t- ( optional ) whether to parse the directory tree file name from the data block file name\n",
PACK_COMMAND,
PACK_COMMAND, // Pack parameters:
"locale", "context", "levelName", "workspacePath", "buildPath",
"numThreads", // Num helper threads.
-1, LZHAM_MAX_HELPER_THREADS, -1,
"compressLevel", // Compress level.
"fastest", "faster", "default", "better", "uber",
UNPACK_COMMAND,
UNPACK_COMMAND,// Unpack parameters:
"fileName", "inputDir", "sanitize"
);
@ -114,10 +122,13 @@ static void ReVPK_Pack(const CCommand& args)
CPackedStoreBuilder builder;
builder.InitLzCompParams();
builder.InitLzEncoder(
argCount > 7 ? (std::min)(atoi(args.Arg(7)), LZHAM_MAX_HELPER_THREADS) : -1, // Num threads.
argCount > 8 ? args.Arg(8) : "default"); // Compress level.
builder.PackWorkspace(pair,
argCount > 5 ? args.Arg(5) : "ship/",
argCount > 6 ? args.Arg(6) : "vpk/");
argCount > 5 ? args.Arg(5) : "ship/", // Workspace path.
argCount > 6 ? args.Arg(6) : "vpk/"); // build path.
timer.End();
Msg(eDLL_T::FS, "*** Time elapsed: '%lf' seconds\n", timer.GetDuration().GetSeconds());
@ -147,7 +158,7 @@ static void ReVPK_Unpack(const CCommand& args)
CPackedStoreBuilder builder;
builder.InitLzDecompParams();
builder.InitLzDecoder();
builder.UnpackWorkspace(vpk, argCount > 3 ? args.Arg(3) : "ship/");
timer.End();

36
src/vpklib/packedstore.cpp
View File

@ -61,31 +61,31 @@ static lzham_compress_level DetermineCompressionLevel(const char* compressionLev
//-----------------------------------------------------------------------------
// Purpose: initialize parameters for compression algorithm
//-----------------------------------------------------------------------------
void CPackedStoreBuilder::InitLzCompParams(const char* compressionLevel, const lzham_int32 maxHelperThreads)
void CPackedStoreBuilder::InitLzEncoder(const lzham_int32 maxHelperThreads, const char* compressionLevel)
{
/*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/
m_lzCompParams.m_struct_size = sizeof(lzham_compress_params);
m_lzCompParams.m_dict_size_log2 = VPK_DICT_SIZE;
m_lzCompParams.m_level = DetermineCompressionLevel(compressionLevel);
m_lzCompParams.m_max_helper_threads = maxHelperThreads;
m_lzCompParams.m_cpucache_total_lines = NULL;
m_lzCompParams.m_cpucache_line_size = NULL;
m_lzCompParams.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING;
m_lzCompParams.m_num_seed_bytes = NULL;
m_lzCompParams.m_pSeed_bytes = NULL;
m_Encoder.m_struct_size = sizeof(lzham_compress_params);
m_Encoder.m_dict_size_log2 = VPK_DICT_SIZE;
m_Encoder.m_level = DetermineCompressionLevel(compressionLevel);
m_Encoder.m_max_helper_threads = maxHelperThreads;
m_Encoder.m_cpucache_total_lines = NULL;
m_Encoder.m_cpucache_line_size = NULL;
m_Encoder.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING;
m_Encoder.m_num_seed_bytes = NULL;
m_Encoder.m_pSeed_bytes = NULL;
}
//-----------------------------------------------------------------------------
// Purpose: initialize parameters for decompression algorithm
//-----------------------------------------------------------------------------
void CPackedStoreBuilder::InitLzDecompParams(void)
void CPackedStoreBuilder::InitLzDecoder(void)
{
/*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/
m_lzDecompParams.m_struct_size = sizeof(lzham_decompress_params);
m_lzDecompParams.m_dict_size_log2 = VPK_DICT_SIZE;
m_lzDecompParams.m_decompress_flags = lzham_decompress_flags::LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED;
m_lzDecompParams.m_num_seed_bytes = NULL;
m_lzDecompParams.m_pSeed_bytes = NULL;
m_Decoder.m_struct_size = sizeof(lzham_decompress_params);
m_Decoder.m_dict_size_log2 = VPK_DICT_SIZE;
m_Decoder.m_decompress_flags = lzham_decompress_flags::LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED;
m_Decoder.m_num_seed_bytes = NULL;
m_Decoder.m_pSeed_bytes = NULL;
}
//-----------------------------------------------------------------------------
@ -534,7 +534,7 @@ void CPackedStoreBuilder::PackWorkspace(const VPKPair_t& vpkPair, const char* wo
if (entryValue.m_bUseCompression)
{
lzham_compress_status_t lzCompStatus = lzham_compress_memory(&m_lzCompParams, pEntryBuffer.get(), &descriptor.m_nCompressedSize, pEntryBuffer.get(),
lzham_compress_status_t lzCompStatus = lzham_compress_memory(&m_Encoder, pEntryBuffer.get(), &descriptor.m_nCompressedSize, pEntryBuffer.get(),
descriptor.m_nUncompressedSize, nullptr);
if (lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
@ -656,7 +656,7 @@ void CPackedStoreBuilder::UnpackWorkspace(const VPKDir_t& vpkDir, const char* wo
if (fragment.m_nCompressedSize > nDstLen)
break; // Corrupt or invalid chunk descriptor.
lzham_decompress_status_t lzDecompStatus = lzham_decompress_memory(&m_lzDecompParams, pDestBuffer.get(),
lzham_decompress_status_t lzDecompStatus = lzham_decompress_memory(&m_Decoder, pDestBuffer.get(),
&nDstLen, pSourceBuffer.get(), fragment.m_nCompressedSize, nullptr);
if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS)

8
src/vpklib/packedstore.h
View File

@ -196,8 +196,8 @@ struct VPKPair_t
class CPackedStoreBuilder
{
public:
void InitLzCompParams(const char* compressionLevel = "default", const lzham_int32 maxHelperThreads = -1);
void InitLzDecompParams(void);
void InitLzEncoder(const lzham_int32 maxHelperThreads = -1, const char* compressionLevel = "default");
void InitLzDecoder(void);
bool Deduplicate(const uint8_t* pEntryBuffer, VPKChunkDescriptor_t& descriptor, const size_t chunkIndex);
@ -205,8 +205,8 @@ public:
void UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspaceName = "");
private:
lzham_compress_params m_lzCompParams; // LZham compression parameters.
lzham_decompress_params m_lzDecompParams; // LZham decompression parameters.
lzham_compress_params m_Encoder; // LZham compression parameters.
lzham_decompress_params m_Decoder; // LZham decompression parameters.
std::unordered_map<string, const VPKChunkDescriptor_t&> m_ChunkHashMap;
};
///////////////////////////////////////////////////////////////////////////////