RTech: add simple pak encoder command

Encodes a pak file with ZSTD.

r5dev/common/callback.cpp

@@ -26,6 +26,7 @@
 #include "engine/server/server.h"
 #endif // !CLIENT_DLL
 
+#include "rtech/pak/pakencode.h"
 #include "rtech/pak/pakdecode.h"
 #include "rtech/pak/pakparse.h"
 #include "rtech/pak/pakstate.h"
@@ -632,6 +633,37 @@ void Pak_Decompress_f(const CCommand& args)
 	FileSystem()->Close(hDecompFile);
 }
 
+/*
+=====================
+RTech_Compress_f
+
+  Compresses input RPak file and
+  dumps results to base path
+=====================
+*/
+void Pak_Compress_f(const CCommand& args)
+{
+	if (args.ArgC() < 2)
+	{
+		return;
+	}
+
+	CUtlString inPakFile;
+	CUtlString outPakFile;
+
+	inPakFile.Format(PLATFORM_PAK_OVERRIDE_PATH "%s", args.Arg(1));
+	outPakFile.Format(PLATFORM_PAK_PATH "%s", args.Arg(1));
+
+	// NULL means default compress level
+	const int compressLevel = args.ArgC() > 2 ? atoi(args.Arg(2)) : NULL;
+
+	if (!Pak_EncodePakFile(inPakFile.String(), outPakFile.String(), compressLevel))
+	{
+		Error(eDLL_T::RTECH, NO_ERROR, "%s - compression failed for '%s'!\n",
+			__FUNCTION__, inPakFile.String());
+	}
+}
+
 /*
 =====================
 VPK_Pack_f

r5dev/common/callback.h

@@ -31,6 +31,7 @@ void Pak_RequestLoad_f(const CCommand& args);
 void Pak_Swap_f(const CCommand& args);
 void Pak_StringToGUID_f(const CCommand& args);
 void Pak_Decompress_f(const CCommand& args);
+void Pak_Compress_f(const CCommand& args);
 void VPK_Pack_f(const CCommand& args);
 void VPK_Unpack_f(const CCommand& args);
 void VPK_Mount_f(const CCommand& args);

r5dev/common/completion.cpp

@@ -170,6 +170,18 @@ int RTech_PakUnload_f_CompletionFunc(char const* partial, char commands[COMMAND_
 	return _Host_Pak_f_CompletionFunc(&s_PakUnloadAutoFileList, partial, commands);
 }
 
+static CBaseAutoCompleteFileList s_PakCompress("pak_compress", "paks/Win64_override", "rpak");
+//-----------------------------------------------------------------------------
+// Purpose: 
+// Input  : *partial - 
+//			**commands - 
+// Output : int
+//-----------------------------------------------------------------------------
+int RTech_PakCompress_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLETION_MAXITEMS][COMMAND_COMPLETION_ITEM_LENGTH])
+{
+	return _Host_Pak_f_CompletionFunc(&s_PakCompress, partial, commands);
+}
+
 static CBaseAutoCompleteFileList s_PakDecompress("pak_decompress", "paks/Win64", "rpak");
 //-----------------------------------------------------------------------------
 // Purpose: 

r5dev/common/completion.h

@@ -11,6 +11,7 @@ int Game_Give_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLE
 
 int RTech_PakLoad_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLETION_MAXITEMS][COMMAND_COMPLETION_ITEM_LENGTH]);
 int RTech_PakUnload_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLETION_MAXITEMS][COMMAND_COMPLETION_ITEM_LENGTH]);
+int RTech_PakCompress_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLETION_MAXITEMS][COMMAND_COMPLETION_ITEM_LENGTH]);
 int RTech_PakDecompress_f_CompletionFunc(char const* partial, char commands[COMMAND_COMPLETION_MAXITEMS][COMMAND_COMPLETION_ITEM_LENGTH]);
 
 inline int(*CBaseAutoCompleteFileList__AutoCompletionFunc)

r5dev/common/global.cpp

@@ -771,6 +771,7 @@ void ConCommand_StaticInit(void)
 	//-------------------------------------------------------------------------
 	// RTECH API                                                              |
 	ConCommand::StaticCreate("pak_strtoguid", "Calculates the GUID from input data.", nullptr, FCVAR_DEVELOPMENTONLY, Pak_StringToGUID_f, nullptr);
+	ConCommand::StaticCreate("pak_compress", "Compresses specified RPAK file.", nullptr, FCVAR_DEVELOPMENTONLY, Pak_Compress_f, RTech_PakCompress_f_CompletionFunc);
 	ConCommand::StaticCreate("pak_decompress", "Decompresses specified RPAK file.", nullptr, FCVAR_DEVELOPMENTONLY, Pak_Decompress_f, RTech_PakDecompress_f_CompletionFunc);
 	ConCommand::StaticCreate("pak_requestload", "Requests asynchronous load for specified RPAK file.", nullptr, FCVAR_DEVELOPMENTONLY, Pak_RequestLoad_f, RTech_PakLoad_f_CompletionFunc);
 	ConCommand::StaticCreate("pak_requestunload", "Requests unload for specified RPAK file or ID.", nullptr, FCVAR_DEVELOPMENTONLY, Pak_RequestUnload_f, RTech_PakUnload_f_CompletionFunc);

r5dev/rtech/CMakeLists.txt

@@ -15,6 +15,9 @@ add_sources( SOURCE_GROUP "Pak"
    "pak/pakalloc.cpp"
    "pak/pakalloc.h"
 
+   "pak/pakencode.cpp"
+   "pak/pakencode.h"
+
    "pak/pakdecode.cpp"
    "pak/pakdecode.h"
 

r5dev/rtech/pak/pakdecode.cpp

@@ -501,7 +501,7 @@ LABEL_69:
 //-----------------------------------------------------------------------------
 // checks if we have enough output buffer room to decode the data stream
 //-----------------------------------------------------------------------------
-uint64_t Pak_HasEnoughOutputRoom(PakDecoder_t* const decoder, const size_t outLen)
+uint64_t Pak_HasEnoughDecodeBufferLeft(PakDecoder_t* const decoder, const size_t outLen)
 {
 	const uint64_t bytesWritten = (decoder->outBufBytePos & ~decoder->outputInvMask);
 
@@ -513,7 +513,7 @@ uint64_t Pak_HasEnoughOutputRoom(PakDecoder_t* const decoder, const size_t outLe
 
 bool Pak_ZStreamDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen)
 {
-	if (!Pak_HasEnoughOutputRoom(decoder, outLen))
+	if (!Pak_HasEnoughDecodeBufferLeft(decoder, outLen))
 		return false;
 
 	size_t inBufLen = inLen;

r5dev/rtech/pak/pakencode.cpp

@@ -0,0 +1,110 @@
+//=============================================================================//
+//
+// Purpose: buffered pak encoder
+//
+//=============================================================================//
+#include "thirdparty/zstd/zstd.h"
+#include "rtech/ipakfile.h"
+#include "pakencode.h"
+#include <filesystem/filesystem.h>
+
+extern CFileSystem_Stdio* FileSystem();
+
+//-----------------------------------------------------------------------------
+// encodes the pak file from buffer, we can't do streamed compression as we
+// need to know the actual decompress size ahead of time, else the runtime will
+// fail as we wouldn't be able to parse the decompressed size from the frame
+// header
+//-----------------------------------------------------------------------------
+bool Pak_EncodePak(const uint8_t* const inBuf, const uint64_t inLen,
+	uint8_t* const outBuf, const uint64_t outLen, const int level)
+{
+	// offset to the actual pak data, the main file header shouldn't be
+	// compressed
+	const size_t dataOffset = sizeof(PakFileHeader_t);
+
+	const size_t compressSize = ZSTD_compress(
+		outBuf + dataOffset,
+		outLen - dataOffset,
+		inBuf + dataOffset,
+		inLen - dataOffset,
+		level);
+
+	if (ZSTD_isError(compressSize))
+		return false;
+
+	const PakFileHeader_t* const inHeader = reinterpret_cast<const PakFileHeader_t* const>(inBuf);
+	PakFileHeader_t* const outHeader = reinterpret_cast<PakFileHeader_t* const>(outBuf);
+
+	// copy the header over
+	*outHeader = *inHeader;
+
+	// the compressed size includes the entire buffer, even the data we didn't
+	// compress like the file header
+	outHeader->compressedSize = compressSize + dataOffset;
+
+	// these flags are required for the game's runtime to decide whether or not to
+	// decompress the pak, and how; see Pak_ProcessPakFile() for more details
+	outHeader->flags |= PAK_HEADER_FLAGS_COMPRESSED;
+	outHeader->flags |= PAK_HEADER_FLAGS_ZSTD;
+
+	return true;
+}
+
+//-----------------------------------------------------------------------------
+// encodes the pak file from file name
+//-----------------------------------------------------------------------------
+bool Pak_EncodePakFile(const char* const inPakFile, const char* const outPakFile, const int level)
+{
+	FileHandle_t inFileHandle = FileSystem()->Open(inPakFile, "rb", "GAME");
+
+	// unable to open
+	if (!inFileHandle)
+		return false;
+
+	const ssize_t fileSize = FileSystem()->Size(inFileHandle);
+
+	// corrupted/truncated
+	if (fileSize <= sizeof(PakFileHeader_t))
+	{
+		FileSystem()->Close(inFileHandle);
+		return false;
+	}
+
+	std::unique_ptr<uint8_t[]> pakBuf(new uint8_t[fileSize]);
+	uint8_t* const pPakBuf = pakBuf.get();
+
+	FileSystem()->Read(pPakBuf, fileSize, inFileHandle);
+	FileSystem()->Close(inFileHandle);
+
+	const PakFileHeader_t* inPakHeader = reinterpret_cast<PakFileHeader_t* const>(pPakBuf);
+
+	// incompatible pak, or not a pak
+	if (inPakHeader->magic != PAK_HEADER_MAGIC || inPakHeader->version != PAK_HEADER_VERSION)
+		return false;
+
+	// already compressed
+	if (inPakHeader->IsCompressed())
+		return false;
+
+	std::unique_ptr<uint8_t[]> outBuf(new uint8_t[inPakHeader->decompressedSize]);
+	uint8_t* const pOutBuf = outBuf.get();
+
+	// encoding failed
+	if (!Pak_EncodePak(pPakBuf, fileSize, pOutBuf, inPakHeader->decompressedSize, level))
+		return false;
+
+	FileHandle_t outFileHandle = FileSystem()->Open(outPakFile, "wb", "GAME");
+
+	// failure to open output file
+	if (!outFileHandle)
+		return false;
+
+	const PakFileHeader_t* outPakHeader = reinterpret_cast<PakFileHeader_t* const>(pOutBuf);
+
+	// this will be true if the entire buffer has been written
+	const bool ret = FileSystem()->Write(pOutBuf, outPakHeader->compressedSize, outFileHandle) == (ssize_t)outPakHeader->compressedSize;
+	FileSystem()->Close(inFileHandle);
+
+	return ret;
+}

r5dev/rtech/pak/pakencode.h

@@ -0,0 +1,9 @@
+#ifndef RTECH_PAKENCODE_H
+#define RTECH_PAKENCODE_H
+
+bool Pak_EncodePak(const uint8_t* const inBuf, const uint64_t inLen,
+	uint8_t* const outBuf, const uint64_t outLen, const int level);
+
+bool Pak_EncodePakFile(const char* const inPakFile, const char* const outPakFile, const int level);
+
+#endif // RTECH_PAKENCODE_H