R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

ReVPK: add option for num helper threads and compress levels

Browse Source 
Allow the user to set the # amount of helper threads and the compress level of the VPK's.
This commit is contained in:
Kawe Mazidjatari 2024-01-13 14:02:46 +01:00
parent 49b82dc61c
commit 0dd630e13a
4 changed files with 47 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/common/callback.cpp
View File

@ -650,7 +650,7 @@ void VPK_Pack_f(const CCommand& args)
CPackedStoreBuilder builder; CPackedStoreBuilder builder;
builder.InitLzCompParams(fs_packedstore_compression_level->GetString(), fs_packedstore_max_helper_threads->GetInt()); builder.InitLzEncoder(fs_packedstore_max_helper_threads->GetInt(), fs_packedstore_compression_level->GetString());
builder.PackWorkspace(pair, fs_packedstore_workspace->GetString(), "vpk/"); builder.PackWorkspace(pair, fs_packedstore_workspace->GetString(), "vpk/");
timer.End(); timer.End();
@ -682,7 +682,7 @@ void VPK_Unpack_f(const CCommand& args)
CPackedStoreBuilder builder; CPackedStoreBuilder builder;
builder.InitLzDecompParams(); builder.InitLzDecoder();
builder.UnpackWorkspace(vpk, fs_packedstore_workspace->GetString()); builder.UnpackWorkspace(vpk, fs_packedstore_workspace->GetString());
timer.End(); timer.End();

35
src/revpk/revpk.cpp
View File

@ -73,20 +73,28 @@ static void ReVPK_Usage()
"ReVPK instructions and options:\n" "ReVPK instructions and options:\n"
"For packing; run 'revpk %s' with the following parameters:\n" "For packing; run 'revpk %s' with the following parameters:\n"
"\t<%s>\t- locale prefix for the directory tree file\n" "\t<%s>\t- locale prefix for the directory tree file\n"
"\t<%s>\t- whether to build for 'server' or 'client'\n" "\t<%s>\t- context scope for the VPK files [\"server\", \"client\"]\n"
"\t<%s>\t- the level name for the VPK files\n" "\t<%s>\t- level name for the VPK files\n"
"\t<%s>\t- ( optional ) path to the workspace containing the manifest file\n" "\t<%s>\t- ( optional ) path to the workspace containing the control file\n"
"\t<%s>\t- ( optional ) path in which the VPK files will be built\n\n" "\t<%s>\t- ( optional ) path in which the VPK files will be built\n"
"\t<%s>\t- ( optional ) max LZHAM helper threads [\"%d\", \"%d\"] \"%d\" ( default ) for max practical\n"
"\t<%s>\t- ( optional ) the level of compression [\"%s\", \"%s\", \"%s\", \"%s\", \"%s\"]\n\n"
"For unpacking; run 'revpk %s' with the following parameters:\n" "For unpacking; run 'revpk %s' with the following parameters:\n"
"\t<%s>\t- path to the directory tree file\n" "\t<%s>\t- name of the target directory tree file\n"
"\t<%s>\t- ( optional ) path to the target directory tree file\n" "\t<%s>\t- ( optional ) path to directory containing the target directory tree file\n"
"\t<%s>\t- ( optional ) whether to parse the directory tree file name from the data block file name\n", "\t<%s>\t- ( optional ) whether to parse the directory tree file name from the data block file name\n",
PACK_COMMAND, PACK_COMMAND, // Pack parameters:
"locale", "context", "levelName", "workspacePath", "buildPath", "locale", "context", "levelName", "workspacePath", "buildPath",
"numThreads", // Num helper threads.
-1, LZHAM_MAX_HELPER_THREADS, -1,
"compressLevel", // Compress level.
"fastest", "faster", "default", "better", "uber",
UNPACK_COMMAND, UNPACK_COMMAND,// Unpack parameters:
"fileName", "inputDir", "sanitize" "fileName", "inputDir", "sanitize"
); );
@ -114,10 +122,13 @@ static void ReVPK_Pack(const CCommand& args)
CPackedStoreBuilder builder; CPackedStoreBuilder builder;
builder.InitLzCompParams(); builder.InitLzEncoder(
argCount > 7 ? (std::min)(atoi(args.Arg(7)), LZHAM_MAX_HELPER_THREADS) : -1, // Num threads.
argCount > 8 ? args.Arg(8) : "default"); // Compress level.
builder.PackWorkspace(pair, builder.PackWorkspace(pair,
argCount > 5 ? args.Arg(5) : "ship/", argCount > 5 ? args.Arg(5) : "ship/", // Workspace path.
argCount > 6 ? args.Arg(6) : "vpk/"); argCount > 6 ? args.Arg(6) : "vpk/"); // build path.
timer.End(); timer.End();
Msg(eDLL_T::FS, "*** Time elapsed: '%lf' seconds\n", timer.GetDuration().GetSeconds()); Msg(eDLL_T::FS, "*** Time elapsed: '%lf' seconds\n", timer.GetDuration().GetSeconds());
@ -147,7 +158,7 @@ static void ReVPK_Unpack(const CCommand& args)
CPackedStoreBuilder builder; CPackedStoreBuilder builder;
builder.InitLzDecompParams(); builder.InitLzDecoder();
builder.UnpackWorkspace(vpk, argCount > 3 ? args.Arg(3) : "ship/"); builder.UnpackWorkspace(vpk, argCount > 3 ? args.Arg(3) : "ship/");
timer.End(); timer.End();

36
src/vpklib/packedstore.cpp
View File

@ -61,31 +61,31 @@ static lzham_compress_level DetermineCompressionLevel(const char* compressionLev
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: initialize parameters for compression algorithm // Purpose: initialize parameters for compression algorithm
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
void CPackedStoreBuilder::InitLzCompParams(const char* compressionLevel, const lzham_int32 maxHelperThreads) void CPackedStoreBuilder::InitLzEncoder(const lzham_int32 maxHelperThreads, const char* compressionLevel)
{ {
/*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ /*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/
m_lzCompParams.m_struct_size = sizeof(lzham_compress_params); m_Encoder.m_struct_size = sizeof(lzham_compress_params);
m_lzCompParams.m_dict_size_log2 = VPK_DICT_SIZE; m_Encoder.m_dict_size_log2 = VPK_DICT_SIZE;
m_lzCompParams.m_level = DetermineCompressionLevel(compressionLevel); m_Encoder.m_level = DetermineCompressionLevel(compressionLevel);
m_lzCompParams.m_max_helper_threads = maxHelperThreads; m_Encoder.m_max_helper_threads = maxHelperThreads;
m_lzCompParams.m_cpucache_total_lines = NULL; m_Encoder.m_cpucache_total_lines = NULL;
m_lzCompParams.m_cpucache_line_size = NULL; m_Encoder.m_cpucache_line_size = NULL;
m_lzCompParams.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING; m_Encoder.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING;
m_lzCompParams.m_num_seed_bytes = NULL; m_Encoder.m_num_seed_bytes = NULL;
m_lzCompParams.m_pSeed_bytes = NULL; m_Encoder.m_pSeed_bytes = NULL;
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: initialize parameters for decompression algorithm // Purpose: initialize parameters for decompression algorithm
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
void CPackedStoreBuilder::InitLzDecompParams(void) void CPackedStoreBuilder::InitLzDecoder(void)
{ {
/*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ /*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/
m_lzDecompParams.m_struct_size = sizeof(lzham_decompress_params); m_Decoder.m_struct_size = sizeof(lzham_decompress_params);
m_lzDecompParams.m_dict_size_log2 = VPK_DICT_SIZE; m_Decoder.m_dict_size_log2 = VPK_DICT_SIZE;
m_lzDecompParams.m_decompress_flags = lzham_decompress_flags::LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED; m_Decoder.m_decompress_flags = lzham_decompress_flags::LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED;
m_lzDecompParams.m_num_seed_bytes = NULL; m_Decoder.m_num_seed_bytes = NULL;
m_lzDecompParams.m_pSeed_bytes = NULL; m_Decoder.m_pSeed_bytes = NULL;
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
@ -534,7 +534,7 @@ void CPackedStoreBuilder::PackWorkspace(const VPKPair_t& vpkPair, const char* wo
if (entryValue.m_bUseCompression) if (entryValue.m_bUseCompression)
{ {
lzham_compress_status_t lzCompStatus = lzham_compress_memory(&m_lzCompParams, pEntryBuffer.get(), &descriptor.m_nCompressedSize, pEntryBuffer.get(), lzham_compress_status_t lzCompStatus = lzham_compress_memory(&m_Encoder, pEntryBuffer.get(), &descriptor.m_nCompressedSize, pEntryBuffer.get(),
descriptor.m_nUncompressedSize, nullptr); descriptor.m_nUncompressedSize, nullptr);
if (lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS) if (lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
@ -656,7 +656,7 @@ void CPackedStoreBuilder::UnpackWorkspace(const VPKDir_t& vpkDir, const char* wo
if (fragment.m_nCompressedSize > nDstLen) if (fragment.m_nCompressedSize > nDstLen)
break; // Corrupt or invalid chunk descriptor. break; // Corrupt or invalid chunk descriptor.
lzham_decompress_status_t lzDecompStatus = lzham_decompress_memory(&m_lzDecompParams, pDestBuffer.get(), lzham_decompress_status_t lzDecompStatus = lzham_decompress_memory(&m_Decoder, pDestBuffer.get(),
&nDstLen, pSourceBuffer.get(), fragment.m_nCompressedSize, nullptr); &nDstLen, pSourceBuffer.get(), fragment.m_nCompressedSize, nullptr);
if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS) if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS)

8
src/vpklib/packedstore.h
View File

@ -196,8 +196,8 @@ struct VPKPair_t
class CPackedStoreBuilder class CPackedStoreBuilder
{ {
public: public:
void InitLzCompParams(const char* compressionLevel = "default", const lzham_int32 maxHelperThreads = -1); void InitLzEncoder(const lzham_int32 maxHelperThreads = -1, const char* compressionLevel = "default");
void InitLzDecompParams(void); void InitLzDecoder(void);
bool Deduplicate(const uint8_t* pEntryBuffer, VPKChunkDescriptor_t& descriptor, const size_t chunkIndex); bool Deduplicate(const uint8_t* pEntryBuffer, VPKChunkDescriptor_t& descriptor, const size_t chunkIndex);
@ -205,8 +205,8 @@ public:
void UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspaceName = ""); void UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspaceName = "");
private: private:
lzham_compress_params m_lzCompParams; // LZham compression parameters. lzham_compress_params m_Encoder; // LZham compression parameters.
lzham_decompress_params m_lzDecompParams; // LZham decompression parameters. lzham_decompress_params m_Decoder; // LZham decompression parameters.
std::unordered_map<string, const VPKChunkDescriptor_t&> m_ChunkHashMap; std::unordered_map<string, const VPKChunkDescriptor_t&> m_ChunkHashMap;
}; };
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////