From 7a03238fc9207a773db293b5134cc47311aa46f9 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Thu, 25 Jan 2024 00:04:39 +0100 Subject: [PATCH] RTech: add simple pak encoder command Encodes a pak file with ZSTD. --- r5dev/common/callback.cpp | 32 ++++++++++ r5dev/common/callback.h | 1 + r5dev/common/completion.cpp | 12 ++++ r5dev/common/completion.h | 1 + r5dev/common/global.cpp | 1 + r5dev/rtech/CMakeLists.txt | 3 + r5dev/rtech/pak/pakdecode.cpp | 4 +- r5dev/rtech/pak/pakencode.cpp | 110 ++++++++++++++++++++++++++++++++++ r5dev/rtech/pak/pakencode.h | 9 +++ 9 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 r5dev/rtech/pak/pakencode.cpp create mode 100644 r5dev/rtech/pak/pakencode.h diff --git a/r5dev/common/callback.cpp b/r5dev/common/callback.cpp index d3c0b149..0023893d 100644 --- a/r5dev/common/callback.cpp +++ b/r5dev/common/callback.cpp @@ -26,6 +26,7 @@ #include "engine/server/server.h" #endif // !CLIENT_DLL +#include "rtech/pak/pakencode.h" #include "rtech/pak/pakdecode.h" #include "rtech/pak/pakparse.h" #include "rtech/pak/pakstate.h" @@ -632,6 +633,37 @@ void Pak_Decompress_f(const CCommand& args) FileSystem()->Close(hDecompFile); } +/* +===================== +RTech_Compress_f + + Compresses input RPak file and + dumps results to base path +===================== +*/ +void Pak_Compress_f(const CCommand& args) +{ + if (args.ArgC() < 2) + { + return; + } + + CUtlString inPakFile; + CUtlString outPakFile; + + inPakFile.Format(PLATFORM_PAK_OVERRIDE_PATH "%s", args.Arg(1)); + outPakFile.Format(PLATFORM_PAK_PATH "%s", args.Arg(1)); + + // NULL means default compress level + const int compressLevel = args.ArgC() > 2 ? atoi(args.Arg(2)) : NULL; + + if (!Pak_EncodePakFile(inPakFile.String(), outPakFile.String(), compressLevel)) + { + Error(eDLL_T::RTECH, NO_ERROR, "%s - compression failed for '%s'!\n", + __FUNCTION__, inPakFile.String()); + } +} + /* ===================== VPK_Pack_f diff --git a/r5dev/common/callback.h b/r5dev/common/callback.h index 912b9f6d..eb4493e9 100644 --- a/r5dev/common/callback.h +++ b/r5dev/common/callback.h @@ -31,6 +31,7 @@ void Pak_RequestLoad_f(const CCommand& args); void Pak_Swap_f(const CCommand& args); void Pak_StringToGUID_f(const CCommand& args); void Pak_Decompress_f(const CCommand& args); +void Pak_Compress_f(const CCommand& args); void VPK_Pack_f(const CCommand& args); void VPK_Unpack_f(const CCommand& args); void VPK_Mount_f(const CCommand& args); diff --git a/r5dev/common/completion.cpp b/r5dev/common/completion.cpp index 702a487c..fb5487d7 100644 --- a/r5dev/common/completion.cpp +++ b/r5dev/common/completion.cpp @@ -170,6 +170,18 @@ int RTech_PakUnload_f_CompletionFunc(char const* partial, char commands[COMMAND_ return _Host_Pak_f_CompletionFunc(&s_PakUnloadAutoFileList, partial, commands); } +static CBaseAutoCompleteFileList s_PakCompress("pak_compress", "paks/Win64_override", "rpak"); +//----------------------------------------------------------------------------- +// Purpose: +// Input : *partial - +// **commands - +// Output : int +//----------------------------------------------------------------------------- +int RTech_PakCompress_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLETION_MAXITEMS][COMMAND_COMPLETION_ITEM_LENGTH]) +{ + return _Host_Pak_f_CompletionFunc(&s_PakCompress, partial, commands); +} + static CBaseAutoCompleteFileList s_PakDecompress("pak_decompress", "paks/Win64", "rpak"); //----------------------------------------------------------------------------- // Purpose: diff --git a/r5dev/common/completion.h b/r5dev/common/completion.h index d34d2d82..583394f1 100644 --- a/r5dev/common/completion.h +++ b/r5dev/common/completion.h @@ -11,6 +11,7 @@ int Game_Give_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLE int RTech_PakLoad_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLETION_MAXITEMS][COMMAND_COMPLETION_ITEM_LENGTH]); int RTech_PakUnload_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLETION_MAXITEMS][COMMAND_COMPLETION_ITEM_LENGTH]); +int RTech_PakCompress_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLETION_MAXITEMS][COMMAND_COMPLETION_ITEM_LENGTH]); int RTech_PakDecompress_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLETION_MAXITEMS][COMMAND_COMPLETION_ITEM_LENGTH]); inline int(*CBaseAutoCompleteFileList__AutoCompletionFunc) diff --git a/r5dev/common/global.cpp b/r5dev/common/global.cpp index 5e6969b2..07e8dd56 100644 --- a/r5dev/common/global.cpp +++ b/r5dev/common/global.cpp @@ -771,6 +771,7 @@ void ConCommand_StaticInit(void) //------------------------------------------------------------------------- // RTECH API | ConCommand::StaticCreate("pak_strtoguid", "Calculates the GUID from input data.", nullptr, FCVAR_DEVELOPMENTONLY, Pak_StringToGUID_f, nullptr); + ConCommand::StaticCreate("pak_compress", "Compresses specified RPAK file.", nullptr, FCVAR_DEVELOPMENTONLY, Pak_Compress_f, RTech_PakCompress_f_CompletionFunc); ConCommand::StaticCreate("pak_decompress", "Decompresses specified RPAK file.", nullptr, FCVAR_DEVELOPMENTONLY, Pak_Decompress_f, RTech_PakDecompress_f_CompletionFunc); ConCommand::StaticCreate("pak_requestload", "Requests asynchronous load for specified RPAK file.", nullptr, FCVAR_DEVELOPMENTONLY, Pak_RequestLoad_f, RTech_PakLoad_f_CompletionFunc); ConCommand::StaticCreate("pak_requestunload", "Requests unload for specified RPAK file or ID.", nullptr, FCVAR_DEVELOPMENTONLY, Pak_RequestUnload_f, RTech_PakUnload_f_CompletionFunc); diff --git a/r5dev/rtech/CMakeLists.txt b/r5dev/rtech/CMakeLists.txt index e2a4164f..7dc2d1fd 100644 --- a/r5dev/rtech/CMakeLists.txt +++ b/r5dev/rtech/CMakeLists.txt @@ -15,6 +15,9 @@ add_sources( SOURCE_GROUP "Pak" "pak/pakalloc.cpp" "pak/pakalloc.h" + "pak/pakencode.cpp" + "pak/pakencode.h" + "pak/pakdecode.cpp" "pak/pakdecode.h" diff --git a/r5dev/rtech/pak/pakdecode.cpp b/r5dev/rtech/pak/pakdecode.cpp index 534d59b7..9b300903 100644 --- a/r5dev/rtech/pak/pakdecode.cpp +++ b/r5dev/rtech/pak/pakdecode.cpp @@ -501,7 +501,7 @@ LABEL_69: //----------------------------------------------------------------------------- // checks if we have enough output buffer room to decode the data stream //----------------------------------------------------------------------------- -uint64_t Pak_HasEnoughOutputRoom(PakDecoder_t* const decoder, const size_t outLen) +uint64_t Pak_HasEnoughDecodeBufferLeft(PakDecoder_t* const decoder, const size_t outLen) { const uint64_t bytesWritten = (decoder->outBufBytePos & ~decoder->outputInvMask); @@ -513,7 +513,7 @@ uint64_t Pak_HasEnoughOutputRoom(PakDecoder_t* const decoder, const size_t outLe bool Pak_ZStreamDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen) { - if (!Pak_HasEnoughOutputRoom(decoder, outLen)) + if (!Pak_HasEnoughDecodeBufferLeft(decoder, outLen)) return false; size_t inBufLen = inLen; diff --git a/r5dev/rtech/pak/pakencode.cpp b/r5dev/rtech/pak/pakencode.cpp new file mode 100644 index 00000000..102a2f05 --- /dev/null +++ b/r5dev/rtech/pak/pakencode.cpp @@ -0,0 +1,110 @@ +//=============================================================================// +// +// Purpose: buffered pak encoder +// +//=============================================================================// +#include "thirdparty/zstd/zstd.h" +#include "rtech/ipakfile.h" +#include "pakencode.h" +#include + +extern CFileSystem_Stdio* FileSystem(); + +//----------------------------------------------------------------------------- +// encodes the pak file from buffer, we can't do streamed compression as we +// need to know the actual decompress size ahead of time, else the runtime will +// fail as we wouldn't be able to parse the decompressed size from the frame +// header +//----------------------------------------------------------------------------- +bool Pak_EncodePak(const uint8_t* const inBuf, const uint64_t inLen, + uint8_t* const outBuf, const uint64_t outLen, const int level) +{ + // offset to the actual pak data, the main file header shouldn't be + // compressed + const size_t dataOffset = sizeof(PakFileHeader_t); + + const size_t compressSize = ZSTD_compress( + outBuf + dataOffset, + outLen - dataOffset, + inBuf + dataOffset, + inLen - dataOffset, + level); + + if (ZSTD_isError(compressSize)) + return false; + + const PakFileHeader_t* const inHeader = reinterpret_cast(inBuf); + PakFileHeader_t* const outHeader = reinterpret_cast(outBuf); + + // copy the header over + *outHeader = *inHeader; + + // the compressed size includes the entire buffer, even the data we didn't + // compress like the file header + outHeader->compressedSize = compressSize + dataOffset; + + // these flags are required for the game's runtime to decide whether or not to + // decompress the pak, and how; see Pak_ProcessPakFile() for more details + outHeader->flags |= PAK_HEADER_FLAGS_COMPRESSED; + outHeader->flags |= PAK_HEADER_FLAGS_ZSTD; + + return true; +} + +//----------------------------------------------------------------------------- +// encodes the pak file from file name +//----------------------------------------------------------------------------- +bool Pak_EncodePakFile(const char* const inPakFile, const char* const outPakFile, const int level) +{ + FileHandle_t inFileHandle = FileSystem()->Open(inPakFile, "rb", "GAME"); + + // unable to open + if (!inFileHandle) + return false; + + const ssize_t fileSize = FileSystem()->Size(inFileHandle); + + // corrupted/truncated + if (fileSize <= sizeof(PakFileHeader_t)) + { + FileSystem()->Close(inFileHandle); + return false; + } + + std::unique_ptr pakBuf(new uint8_t[fileSize]); + uint8_t* const pPakBuf = pakBuf.get(); + + FileSystem()->Read(pPakBuf, fileSize, inFileHandle); + FileSystem()->Close(inFileHandle); + + const PakFileHeader_t* inPakHeader = reinterpret_cast(pPakBuf); + + // incompatible pak, or not a pak + if (inPakHeader->magic != PAK_HEADER_MAGIC || inPakHeader->version != PAK_HEADER_VERSION) + return false; + + // already compressed + if (inPakHeader->IsCompressed()) + return false; + + std::unique_ptr outBuf(new uint8_t[inPakHeader->decompressedSize]); + uint8_t* const pOutBuf = outBuf.get(); + + // encoding failed + if (!Pak_EncodePak(pPakBuf, fileSize, pOutBuf, inPakHeader->decompressedSize, level)) + return false; + + FileHandle_t outFileHandle = FileSystem()->Open(outPakFile, "wb", "GAME"); + + // failure to open output file + if (!outFileHandle) + return false; + + const PakFileHeader_t* outPakHeader = reinterpret_cast(pOutBuf); + + // this will be true if the entire buffer has been written + const bool ret = FileSystem()->Write(pOutBuf, outPakHeader->compressedSize, outFileHandle) == (ssize_t)outPakHeader->compressedSize; + FileSystem()->Close(inFileHandle); + + return ret; +} diff --git a/r5dev/rtech/pak/pakencode.h b/r5dev/rtech/pak/pakencode.h new file mode 100644 index 00000000..647e0320 --- /dev/null +++ b/r5dev/rtech/pak/pakencode.h @@ -0,0 +1,9 @@ +#ifndef RTECH_PAKENCODE_H +#define RTECH_PAKENCODE_H + +bool Pak_EncodePak(const uint8_t* const inBuf, const uint64_t inLen, + uint8_t* const outBuf, const uint64_t outLen, const int level); + +bool Pak_EncodePakFile(const char* const inPakFile, const char* const outPakFile, const int level); + +#endif // RTECH_PAKENCODE_H