R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

RTech: add buffered decompress function, cleanup and bug fixes

Browse Source 
The pak_decompress callback has been fully moved to a dedicated buffered pak decoder function, also added descriptive error to the buffered pak encode function. Fixed a bug in the streamed pak decoder where the next required streamed buffer size would always be below actually required on any subsequent patch rpaks that are getting loaded. Also fixed assert in Pak_ZStreamDecode, as the inbuf position can equal total streamed size, but shall never exceed it!
This commit is contained in:
Kawe Mazidjatari 2024-01-28 18:15:19 +01:00
parent c7471dc472
commit 3b9626612f
4 changed files with 283 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
r5dev/common/callback.cpp
View File

@ -9,6 +9,7 @@
#include "windows/id3dx.h"
#include "tier0/fasttimer.h"
#include "tier1/cvar.h"
#include "tier1/fmtstr.h"
#ifndef CLIENT_DLL
#include "engine/server/sv_rcon.h"
#endif // !CLIENT_DLL
@ -485,152 +486,14 @@ void Pak_Decompress_f(const CCommand& args)
return;
}
CUtlString inPakFile;
CUtlString outPakFile;
CFmtStr1024 inPakFile(PLATFORM_PAK_PATH "%s", args.Arg(1));
CFmtStr1024 outPakFile(PLATFORM_PAK_OVERRIDE_PATH "%s", args.Arg(1));
inPakFile.Format(PLATFORM_PAK_PATH "%s", args.Arg(1));
outPakFile.Format(PLATFORM_PAK_OVERRIDE_PATH "%s", args.Arg(1));
Msg(eDLL_T::RTECH, "______________________________________________________________\n");
Msg(eDLL_T::RTECH, "-+ RTech decompress ------------------------------------------\n");
if (!FileSystem()->FileExists(inPakFile.String(), "GAME"))
if (!Pak_DecodePakFile(inPakFile.String(), outPakFile.String()))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s - pak file '%s' does not exist!\n",
Error(eDLL_T::RTECH, NO_ERROR, "%s - decompression failed for '%s'!\n",
__FUNCTION__, inPakFile.String());
return;
}
Msg(eDLL_T::RTECH, " |-+ Processing: '%s'\n", inPakFile.String());
FileHandle_t hPakFile = FileSystem()->Open(inPakFile.String(), "rb", "GAME");
if (!hPakFile)
{
Error(eDLL_T::RTECH, NO_ERROR, "%s - Unable to open '%s' (insufficient rights?)\n",
__FUNCTION__, inPakFile.String());
return;
}
const ssize_t nFileSize = FileSystem()->Size(hPakFile);
std::unique_ptr<uint8_t[]> pPakBufContainer(new uint8_t[nFileSize]);
uint8_t* const pPakBuf = pPakBufContainer.get();
FileSystem()->Read(pPakBuf, nFileSize, hPakFile);
FileSystem()->Close(hPakFile);
PakFileHeader_t* pHeader = reinterpret_cast<PakFileHeader_t*>(pPakBuf);
SYSTEMTIME systemTime;
FileTimeToSystemTime(&pHeader->fileTime, &systemTime);
Msg(eDLL_T::RTECH, " | |-+ Header ------------------------------------------------\n");
Msg(eDLL_T::RTECH, " | |-- Magic : '0x%08X'\n", pHeader->magic);
Msg(eDLL_T::RTECH, " | |-- Version : '%hu'\n", pHeader->version);
Msg(eDLL_T::RTECH, " | |-- Flags : '0x%04hX'\n", pHeader->flags);
Msg(eDLL_T::RTECH, " | |-- Time : '%hu-%hu-%hu/%hu %hu:%hu:%hu.%hu'\n",
systemTime.wYear, systemTime.wMonth, systemTime.wDay, systemTime.wDayOfWeek,
systemTime.wHour, systemTime.wMinute, systemTime.wSecond, systemTime.wMilliseconds);
Msg(eDLL_T::RTECH, " | |-- Hash : '0x%08llX'\n", pHeader->checksum);
Msg(eDLL_T::RTECH, " | |-- Entries : '%u'\n", pHeader->assetCount);
Msg(eDLL_T::RTECH, " | |-+ Compression -----------------------------------------\n");
Msg(eDLL_T::RTECH, " | |-- Size comp: '%zu'\n", pHeader->compressedSize);
Msg(eDLL_T::RTECH, " | |-- Size decp: '%zu'\n", pHeader->decompressedSize);
if (pHeader->magic != PAK_HEADER_MAGIC)
{
Error(eDLL_T::RTECH, NO_ERROR, "%s - pak file '%s' has invalid magic!\n",
__FUNCTION__, inPakFile.String());
return;
}
if (!(pHeader->flags & PAK_HEADER_FLAGS_COMPRESSED))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s - pak file '%s' already decompressed!\n",
__FUNCTION__, inPakFile.String());
return;
}
if (pHeader->compressedSize != size_t(nFileSize))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s - pak file '%s' decompressed size '%zu' doesn't match expected size '%zu'!\n",
__FUNCTION__, inPakFile.String(), size_t(nFileSize), pHeader->compressedSize);
return;
}
const bool usesCustomCompression = pHeader->flags & PAK_HEADER_FLAGS_ZSTREAM;
PakDecoder_t decoder{};
const uint64_t nDecompSize = Pak_InitDecoder(&decoder, pPakBuf, nullptr, UINT64_MAX, UINT64_MAX, nFileSize, NULL, sizeof(PakFileHeader_t), usesCustomCompression);
if (nDecompSize != pHeader->decompressedSize)
{
Error(eDLL_T::RTECH, NO_ERROR, "%s - calculated size: '%llu' expected: '%llu'!\n",
__FUNCTION__, nDecompSize, pHeader->decompressedSize);
return;
}
else
{
Msg(eDLL_T::RTECH, " | |-- Size calc: '%llu'\n", nDecompSize);
}
Msg(eDLL_T::RTECH, " | |-- Ratio : '%.02f'\n", (pHeader->compressedSize * 100.f) / pHeader->decompressedSize);
std::unique_ptr<uint8_t[]> pDecompBufContainer(new uint8_t[pHeader->decompressedSize]);
uint8_t* const pDecompBuf = pDecompBufContainer.get();
// Needs full decode mask as we don't decompress in chunks.
decoder.outputMask = UINT64_MAX;
decoder.outputBuf = pDecompBuf;
uint8_t nDecompResult = Pak_StreamToBufferDecode(&decoder, pHeader->compressedSize, pHeader->decompressedSize, usesCustomCompression);
if (nDecompResult != 1)
{
Error(eDLL_T::RTECH, NO_ERROR, "%s - decompression failed for '%s' return value: '%hu'!\n",
__FUNCTION__, inPakFile.String(), nDecompResult);
}
pHeader->flags &= ~PAK_HEADER_FLAGS_COMPRESSED; // Remove compressed flags.
pHeader->compressedSize = pHeader->decompressedSize; // Equal compressed size with decompressed.
FileSystem()->CreateDirHierarchy(PLATFORM_PAK_OVERRIDE_PATH, "GAME");
FileHandle_t hDecompFile = FileSystem()->Open(outPakFile.String(), "wb", "GAME");
if (!hDecompFile)
{
Error(eDLL_T::RTECH, NO_ERROR, "%s - Unable to write to '%s' (read-only?)\n",
__FUNCTION__, outPakFile.String());
return;
}
if (pHeader->patchIndex > 0) // Check if its an patch rpak.
{
// Loop through all the structs and patch their compress size.
for (uint16_t i = 1, nPatchOffset = (sizeof(PakFileHeader_t) + sizeof(uint64_t));
i <= pHeader->patchIndex; i++, nPatchOffset += sizeof(PakPatchFileHeader_t))
{
PakPatchFileHeader_t* pPatchHeader = reinterpret_cast<PakPatchFileHeader_t*>(pDecompBuf + nPatchOffset);
Msg(eDLL_T::RTECH, " | |-+ Patch #%02hu -----------------------------------------\n", i);
Msg(eDLL_T::RTECH, " | %s |-- Size comp: '%llu'\n", i < pHeader->patchIndex ? "|" : " ", pPatchHeader->m_sizeDisk);
Msg(eDLL_T::RTECH, " | %s |-- Size decp: '%llu'\n", i < pHeader->patchIndex ? "|" : " ", pPatchHeader->m_sizeMemory);
pPatchHeader->m_sizeDisk = pPatchHeader->m_sizeMemory; // Fix size for decompress.
}
}
memcpy_s(pDecompBuf, sizeof(PakFileHeader_t), pPakBuf, sizeof(PakFileHeader_t));// Overwrite first 0x80 bytes which are NULL with the header data.
FileSystem()->Write(pDecompBuf, decoder.decompSize, hDecompFile);
Msg(eDLL_T::RTECH, " |-- Checksum : '0x%08X'\n", crc32::update(NULL, pDecompBuf, decoder.decompSize));
Msg(eDLL_T::RTECH, "-+ Decompressed pak file to: '%s'\n", outPakFile.String());
Msg(eDLL_T::RTECH, "--------------------------------------------------------------\n");
FileSystem()->Close(hDecompFile);
}
/*
@ -648,11 +511,8 @@ void Pak_Compress_f(const CCommand& args)
return;
}
CUtlString inPakFile;
CUtlString outPakFile;
inPakFile.Format(PLATFORM_PAK_OVERRIDE_PATH "%s", args.Arg(1));
outPakFile.Format(PLATFORM_PAK_PATH "%s", args.Arg(1));
CFmtStr1024 inPakFile(PLATFORM_PAK_OVERRIDE_PATH "%s", args.Arg(1));
CFmtStr1024 outPakFile(PLATFORM_PAK_PATH "%s", args.Arg(1));
// NULL means default compress level
const int compressLevel = args.ArgC() > 2 ? atoi(args.Arg(2)) : NULL;

209
r5dev/rtech/pak/pakdecode.cpp
View File

@ -1,12 +1,17 @@
//=============================================================================//
//
// Purpose: streamed & buffered pak file decoder
// Purpose: streamed & buffered pak decoder
//
//=============================================================================//
#include "tier0/binstream.h"
#include "tier1/fmtstr.h"
#include "thirdparty/zstd/zstd.h"
#include "thirdparty/zstd/decompress/zstd_decompress_internal.h"
#include "rtech/ipakfile.h"
#include "paktools.h"
#include "pakdecode.h"
//-----------------------------------------------------------------------------
@ -136,7 +141,7 @@ static const unsigned char /*141313180*/ s_defaultDecoderLUT[] =
bool Pak_HasEnoughDecodeBufferAvailable(PakDecoder_t* const decoder, const size_t outLen)
{
const uint64_t bytesWritten = (decoder->outBufBytePos & ~decoder->outputInvMask);
return (outLen >= decoder->outputInvMask + (bytesWritten + 1) || outLen >= decoder->decompSize);
return (outLen >= decoder->outputInvMask + (bytesWritten +1) || outLen >= decoder->decompSize);
}
//-----------------------------------------------------------------------------
@ -153,14 +158,14 @@ bool Pak_HasEnoughStreamedDataForDecode(PakDecoder_t* const decoder, const size_
// gets the frame for the data in the ring buffer, the frame returned is always
// ending to the end of the ring buffer, or the end of the data itself
//-----------------------------------------------------------------------------
PakRingBufferFrame_t Pak_ComputeRingBufferFrame(const uint64_t bufMask, const size_t seekPos, const size_t dataLen)
PakRingBufferFrame_t Pak_DetermineRingBufferFrame(const uint64_t bufMask, const size_t seekPos, const size_t dataLen)
{
PakRingBufferFrame_t ring;
ring.bufIndex = seekPos & bufMask;
// the total amount of bytes used and available in this frame
const size_t bytesUsed = ring.bufIndex & bufMask;
const size_t totalAvail = bufMask+1 - bytesUsed;
const size_t totalAvail = bufMask +1 - bytesUsed;
// the last part of the data might be smaller than the remainder of the ring
// buffer; clamp it
@ -566,7 +571,7 @@ LABEL_69:
size_t Pak_ZStreamDecoderInit(PakDecoder_t* const decoder, const uint8_t* const fileBuffer,
const uint64_t inputMask, const size_t dataSize, const size_t dataOffset, const size_t headerSize)
{
// NOTE: on original paks, this data is passed out of the frame header,
// NOTE: on original paks, this data is parsed out of the frame header,
// but for ZStd encoded paks we are always limiting this to the ring
// buffer size
decoder->inputInvMask = PAK_DECODE_OUT_RING_BUFFER_MASK;
@ -605,7 +610,7 @@ size_t Pak_ZStreamDecoderInit(PakDecoder_t* const decoder, const uint8_t* const
// we need at least this many bytes of streamed data to process the frame
// header of the compressed block
decoder->bufferSizeNeeded = decoder->headerOffset + decoder->frameHeaderSize;
decoder->bufferSizeNeeded = decoder->inBufBytePos + decoder->frameHeaderSize;
// must include header size
decoder->decompSize = dctx->fParams.frameContentSize + headerSize;
@ -614,16 +619,16 @@ size_t Pak_ZStreamDecoderInit(PakDecoder_t* const decoder, const uint8_t* const
//-----------------------------------------------------------------------------
// decodes the ZStd data stream up to available buffer or data, whichever ends
// first (determined by Pak_ComputeRingBufferFrame())
// first (determined by Pak_DetermineRingBufferFrame())
//-----------------------------------------------------------------------------
bool Pak_ZStreamDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen)
{
// must have a zstream decoder at this point, and input seek pos may not
// exceed or equal inLen as we can't read past currently streamed data;
// this should've been checked before calling this function
assert(decoder->zstreamContext && decoder->inBufBytePos < inLen);
// must have a ZStd decoder at this point, and input seek pos may not exceed
// inLen as we can't read past currently streamed data; this should've been
// checked before calling this function
assert(decoder->zstreamContext && decoder->inBufBytePos <= inLen);
PakRingBufferFrame_t outFrame = Pak_ComputeRingBufferFrame(decoder->outputMask, decoder->outBufBytePos, outLen);
PakRingBufferFrame_t outFrame = Pak_DetermineRingBufferFrame(decoder->outputMask, decoder->outBufBytePos, outLen);
ZSTD_outBuffer outBuffer = {
&decoder->outputBuf[outFrame.bufIndex],
@ -631,7 +636,7 @@ bool Pak_ZStreamDecode(PakDecoder_t* const decoder, const size_t inLen, const si
NULL
};
PakRingBufferFrame_t inFrame = Pak_ComputeRingBufferFrame(decoder->inputMask, decoder->inBufBytePos, inLen);
PakRingBufferFrame_t inFrame = Pak_DetermineRingBufferFrame(decoder->inputMask, decoder->inBufBytePos, inLen);
ZSTD_inBuffer inBuffer = {
&decoder->inputBuf[inFrame.bufIndex],
@ -649,26 +654,17 @@ bool Pak_ZStreamDecode(PakDecoder_t* const decoder, const size_t inLen, const si
return false;
}
// the first call to this function expects a buffer with at least the size
// of the pak file header + the ZStd frame header, after this call we
// should subtract it from the needed buffer as
if (decoder->frameHeaderSize)
{
decoder->bufferSizeNeeded -= decoder->frameHeaderSize;
decoder->frameHeaderSize = 0;
}
// advance buffer io positions, required so the main parser could already
// start parsing the headers while the rest is getting decoded still
decoder->inBufBytePos += inBuffer.pos;
decoder->outBufBytePos += outBuffer.pos;
// on the next call, we need at least this amount of data streamed in order
// to decode the rest of the pak file, as this is where reading has stopped
// this value may equal the currently streamed input size, as its possible
// this function is getting called to flush the remainder decoded data into
// the out buffer which got truncated off on the call prior due to wrapping
decoder->bufferSizeNeeded += inBuffer.pos;
// advance buffer io positions, required so the main parser could already
// start parsing the headers while the rest is getting decoded still
decoder->inBufBytePos += inBuffer.pos;
decoder->outBufBytePos += outBuffer.pos;
decoder->bufferSizeNeeded = decoder->inBufBytePos;
const bool decoded = ret == NULL;
@ -709,8 +705,10 @@ size_t Pak_InitDecoder(PakDecoder_t* const decoder, const uint8_t* const inputBu
decoder->outputMask = outputMask;
// the current positions in the input and output buffers; if we deal with
// paks that are patched, the buffer positions during the init and decode
// call on subsequent patches may not be at the start of the buffers
// paks that are patched, the input buffer position during the init and
// decode call on subsequent patches may not be at the start of the buffer,
// they will end where the previous 'to patch' pak had finished streaming
// and decoding
decoder->inBufBytePos = dataOffset + headerSize;
decoder->outBufBytePos = headerSize;
@ -721,7 +719,9 @@ size_t Pak_InitDecoder(PakDecoder_t* const decoder, const uint8_t* const inputBu
}
//-----------------------------------------------------------------------------
// decodes streamed input pak data
// decodes streamed input pak data; on patched pak files, inLen will continue
// from where the base pak had ended as patch pak files are considered part of
// the pak file that's currently getting loaded
//-----------------------------------------------------------------------------
bool Pak_StreamToBufferDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen, const bool useZStream)
{
@ -736,3 +736,152 @@ bool Pak_StreamToBufferDecode(PakDecoder_t* const decoder, const size_t inLen, c
return Pak_RStreamDecode(decoder, inLen, outLen);
}
//-----------------------------------------------------------------------------
// decodes buffered input pak data
//-----------------------------------------------------------------------------
bool Pak_BufferToBufferDecode(uint8_t* const inBuf, uint8_t* const outBuf, const size_t pakSize)
{
PakFileHeader_t* const inHeader = reinterpret_cast<PakFileHeader_t*>(inBuf);
const bool usesZStream = inHeader->flags & PAK_HEADER_FLAGS_ZSTREAM;
PakDecoder_t decoder{};
const size_t decompressedSize = Pak_InitDecoder(&decoder, inBuf, outBuf, UINT64_MAX, UINT64_MAX, pakSize, NULL, sizeof(PakFileHeader_t), usesZStream);
if (decompressedSize != inHeader->decompressedSize)
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: decompressed size: '%zu' expected: '%zu'!\n",
__FUNCTION__, decompressedSize, inHeader->decompressedSize);
return false;
}
// we should always have enough buffer room at this point
if (!Pak_StreamToBufferDecode(&decoder, inHeader->compressedSize, inHeader->decompressedSize, usesZStream))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: decompression failed!\n",
__FUNCTION__);
return false;
}
PakFileHeader_t* const outHeader = reinterpret_cast<PakFileHeader_t*>(outBuf);
// copy the header over to the decoded buffer
*outHeader = *inHeader;
// remove compress flags
outHeader->flags &= ~PAK_HEADER_FLAGS_COMPRESSED;
outHeader->flags &= ~PAK_HEADER_FLAGS_ZSTREAM;
// equal compressed size with decompressed
outHeader->compressedSize = outHeader->decompressedSize;
return true;
}
//-----------------------------------------------------------------------------
// decodes the pak file from file name
//-----------------------------------------------------------------------------
bool Pak_DecodePakFile(const char* const inPakFile, const char* const outPakFile)
{
// if this path doesn't exist, we must create it first before trying to
// open the out file
if (!Pak_CreateOverridePath())
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: failed to create output path for pak file '%s'!\n",
__FUNCTION__, outPakFile);
return false;
}
CIOStream inPakStream;
if (!inPakStream.Open(inPakFile, CIOStream::READ | CIOStream::BINARY))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: failed to open pak file '%s' for read!\n",
__FUNCTION__, inPakFile);
return false;
}
CIOStream outPakStream;
if (!outPakStream.Open(outPakFile, CIOStream::WRITE | CIOStream::BINARY))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: failed to open pak file '%s' for write!\n",
__FUNCTION__, outPakFile);
return false;
}
const size_t fileSize = inPakStream.GetSize();
if (fileSize <= sizeof(PakFileHeader_t))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: pak '%s' appears truncated!\n",
__FUNCTION__, inPakFile);
return false;
}
std::unique_ptr<uint8_t[]> inPakBufContainer(new uint8_t[fileSize]);
uint8_t* const inPakBuf = inPakBufContainer.get();
inPakStream.Read(inPakBuf, fileSize);
inPakStream.Close();
PakFileHeader_t* const inHeader = reinterpret_cast<PakFileHeader_t*>(inPakBuf);
if (inHeader->magic != PAK_HEADER_MAGIC || inHeader->version != PAK_HEADER_VERSION)
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: pak '%s' has incompatible or invalid header!\n",
__FUNCTION__, inPakFile);
return false;
}
if (!inHeader->IsCompressed())
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: pak '%s' is already decompressed!\n",
__FUNCTION__, inPakFile);
return false;
}
if (inHeader->compressedSize != fileSize)
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: pak '%s' appears truncated or corrupt; compressed size: '%zu' expected: '%zu'!\n",
__FUNCTION__, inPakFile, fileSize, inHeader->compressedSize);
return false;
}
Pak_ShowHeaderDetails(inHeader);
std::unique_ptr<uint8_t[]> outPakBufContainer(new uint8_t[inHeader->decompressedSize]);
uint8_t* const outPakBuf = outPakBufContainer.get();
if (!Pak_BufferToBufferDecode(inPakBuf, outPakBuf, fileSize))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: failed to decompress pak file '%s'!\n",
__FUNCTION__, inPakFile);
return false;
}
PakFileHeader_t* const outHeader = reinterpret_cast<PakFileHeader_t*>(outPakBuf);
// NOTE: if the paks this particular pak patches have different sizes than
// current sizes in the patch header, the runtime will crash!
if (outHeader->patchIndex && !Pak_UpdatePatchHeaders(outPakBuf, outPakFile))
{
Warning(eDLL_T::RTECH, "%s: pak '%s' is a patch pak, but the pak(s) it patches weren't found; patch headers not updated!\n",
__FUNCTION__, inPakFile);
}
outPakStream.Write(outPakBuf, outHeader->decompressedSize);
Msg(eDLL_T::RTECH, "Decompressed pak file to: '%s'\n", outPakFile);
return true;
}

3
r5dev/rtech/pak/pakdecode.h
View File

@ -7,5 +7,8 @@ extern size_t Pak_InitDecoder(PakDecoder_t* const decoder, const uint8_t* const
const size_t headerSize, const bool useZStream);
extern bool Pak_StreamToBufferDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen, const bool useCustom);
extern bool Pak_BufferToBufferDecode(uint8_t* const inBuf, uint8_t* const outBuf, const size_t pakSize);
extern bool Pak_DecodePakFile(const char* const inPakFile, const char* const outPakFile);
#endif // RTECH_PAKDECODE_H

196
r5dev/rtech/pak/pakencode.cpp
View File

@ -30,7 +30,12 @@ bool Pak_BufferToBufferEncode(const uint8_t* const inBuf, const uint64_t inLen,
level);
if (ZSTD_isError(compressSize))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: compression failed! [%s]\n",
__FUNCTION__, ZSTD_getErrorName(compressSize));
return false;
}
PakFileHeader_t* const outHeader = reinterpret_cast<PakFileHeader_t* const>(outBuf);
@ -46,127 +51,114 @@ bool Pak_BufferToBufferEncode(const uint8_t* const inBuf, const uint64_t inLen,
return true;
}
//-----------------------------------------------------------------------------
// searches for pak patches and updates all referenced files in patch headers
//-----------------------------------------------------------------------------
bool Pak_UpdatePatchHeaders(uint8_t* const inBuf, const char* const outPakFile)
{
// file name without extension and patch number ID
char baseFilePath[MAX_PATH];
snprintf(baseFilePath, sizeof(baseFilePath), "%s", outPakFile);
const char* const fileNameUnqualified = V_UnqualifiedFileName(baseFilePath);
V_StripExtension(baseFilePath, baseFilePath, sizeof(baseFilePath));
// strip the patch id, but make sure we only do this on the file name
// and not the path
char* const patchIdentifier = strrchr(baseFilePath, '(');
if (patchIdentifier && patchIdentifier > fileNameUnqualified)
*patchIdentifier = '\0';
// NOTE: we modify the in buffer as the patch headers belong to the
// compressed section
PakFileHeader_t* const inHeader = reinterpret_cast<PakFileHeader_t* const>(inBuf);
// update each patch header
for (uint16_t i = 0; i < inHeader->patchIndex; i++)
{
short patchNumber = Pak_GetPatchNumberForIndex(inHeader, i);
char patchFile[MAX_PATH];
// the first patch number does not have an identifier in its name
if (patchNumber == 0)
snprintf(patchFile, sizeof(patchFile), "%s.rpak", baseFilePath);
else
snprintf(patchFile, sizeof(patchFile), "%s(%02u).rpak", baseFilePath, patchNumber);
CIOStream inPatch;
// unable to open patch while there should be one, we must calculate
// new file sizes here, or else the runtime would fail to load them
if (!inPatch.Open(patchFile, CIOStream::READ | CIOStream::BINARY))
return false;
const size_t fileSize = inPatch.GetSize();
// pak appears truncated
if (fileSize <= sizeof(PakFileHeader_t))
return false;
DevMsg(eDLL_T::RTECH, "%s: updating patch header for pak '%s', new size = %zu\n",
__FUNCTION__, patchFile, fileSize);
PakPatchFileHeader_t* const patchHeader = Pak_GetPatchFileHeader(inHeader, i);
patchHeader->m_sizeDisk = fileSize;
}
return true;
}
//-----------------------------------------------------------------------------
// encodes the pak file from file name
//-----------------------------------------------------------------------------
bool Pak_EncodePakFile(const char* const inPakFile, const char* const outPakFile, const int level)
{
CIOStream inFile;
// failed to open
if (!inFile.Open(inPakFile, CIOStream::READ | CIOStream::BINARY))
return false;
const ssize_t fileSize = inFile.GetSize();
// file appears truncated
if (fileSize <= sizeof(PakFileHeader_t))
return false;
std::unique_ptr<uint8_t[]> pakBuf(new uint8_t[fileSize]);
uint8_t* const pPakBuf = pakBuf.get();
inFile.Read(pPakBuf, fileSize);
inFile.Close();
const PakFileHeader_t* inHeader = reinterpret_cast<PakFileHeader_t* const>(pPakBuf);
// incompatible pak, or not a pak
if (inHeader->magic != PAK_HEADER_MAGIC || inHeader->version != PAK_HEADER_VERSION)
return false;
// already compressed
if (inHeader->IsCompressed())
return false;
if (inHeader->patchIndex && !Pak_UpdatePatchHeaders(pPakBuf, outPakFile))
if (!Pak_CreateBasePath())
{
// pak has patches but one or more weren't found; the headers need to
// be updated with new compression sizes since the runtime uses them to
// check for truncation
Error(eDLL_T::RTECH, NO_ERROR, "%s: failed to create output path for pak file '%s'!\n",
__FUNCTION__, outPakFile);
return false;
}
std::unique_ptr<uint8_t[]> outBuf(new uint8_t[inHeader->decompressedSize]);
uint8_t* const pOutBuf = outBuf.get();
CIOStream inPakStream;
PakFileHeader_t* const outHeader = reinterpret_cast<PakFileHeader_t* const>(pOutBuf);
if (!inPakStream.Open(inPakFile, CIOStream::READ | CIOStream::BINARY))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: failed to open pak file '%s' for read!\n",
__FUNCTION__, inPakFile);
return false;
}
CIOStream outPakStream;
if (!outPakStream.Open(outPakFile, CIOStream::WRITE | CIOStream::BINARY))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: failed to open pak file '%s' for write!\n",
__FUNCTION__, outPakFile);
return false;
}
const size_t fileSize = inPakStream.GetSize();
// file appears truncated
if (fileSize <= sizeof(PakFileHeader_t))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: pak '%s' appears truncated!\n",
__FUNCTION__, inPakFile);
return false;
}
std::unique_ptr<uint8_t[]> inPakBufContainer(new uint8_t[fileSize]);
uint8_t* const inPakBuf = inPakBufContainer.get();
inPakStream.Read(inPakBuf, fileSize);
inPakStream.Close();
const PakFileHeader_t* inHeader = reinterpret_cast<PakFileHeader_t* const>(inPakBuf);
if (inHeader->magic != PAK_HEADER_MAGIC || inHeader->version != PAK_HEADER_VERSION)
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: pak '%s' has incompatible or invalid header!\n",
__FUNCTION__, inPakFile);
return false;
}
if (inHeader->IsCompressed())
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: pak '%s' is already compressed!\n",
__FUNCTION__, inPakFile);
return false;
}
if (inHeader->compressedSize != fileSize)
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: pak '%s' appears truncated or corrupt; compressed size: '%zu' expected: '%zu'!\n",
__FUNCTION__, inPakFile, fileSize, inHeader->compressedSize);
return false;
}
// NOTE: if the paks this particular pak patches have different sizes than
// current sizes in the patch header, the runtime will crash!
if (inHeader->patchIndex && !Pak_UpdatePatchHeaders(inPakBuf, outPakFile))
{
Warning(eDLL_T::RTECH, "%s: pak '%s' is a patch pak, but the pak(s) it patches weren't found; patch headers not updated!\n",
__FUNCTION__, inPakFile);
}
std::unique_ptr<uint8_t[]> outPakBufContainer(new uint8_t[inHeader->decompressedSize]);
uint8_t* const outPakBuf = outPakBufContainer.get();
PakFileHeader_t* const outHeader = reinterpret_cast<PakFileHeader_t* const>(outPakBuf);
// copy the header over
*outHeader = *inHeader;
// encoding failed
if (!Pak_BufferToBufferEncode(pPakBuf, fileSize, pOutBuf, inHeader->decompressedSize, level))
if (!Pak_BufferToBufferEncode(inPakBuf, inHeader->compressedSize, outPakBuf, inHeader->decompressedSize, level))
{
Error(eDLL_T::RTECH, NO_ERROR, "%s: failed to compress pak file '%s'!\n",
__FUNCTION__, inPakFile);
return false;
}
CIOStream outFile;
const PakFileHeader_t* outPakHeader = reinterpret_cast<PakFileHeader_t* const>(outPakBuf);
// failure to open output file
if (!outFile.Open(outPakFile, CIOStream::WRITE | CIOStream::BINARY))
return false;
const PakFileHeader_t* outPakHeader = reinterpret_cast<PakFileHeader_t* const>(pOutBuf);
Pak_ShowHeaderDetails(outPakHeader);
// this will be true if the entire buffer has been written
outFile.Write(pOutBuf, outPakHeader->compressedSize);
outPakStream.Write(outPakBuf, outPakHeader->compressedSize);
Msg(eDLL_T::RTECH, "Compressed pak file to: '%s'\n", outPakFile);
return true;
}