From 6e7bfab4bd5478bc557b84a4666d8ccd4c43dd0a Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Thu, 22 Feb 2024 02:09:28 +0100 Subject: [PATCH] RTech: major cleanup of pak globals and improvement Add missing types to fully mapped out PakGlobals_s struct, which is a 13MiB+ structure! This covers the vast majority of the pakfile system which is why we had to change a bunch of stuff for this patch. This patch also comes with: - Reversed 'JobFifoLock_s' structure - Reversed 'PakTracker_s' structure - Reversed 'PakAssetTracker_s' structure Many globals have been dropped as they were covered by the large PakGlobals_s singleton. The pak decoder logic has been changed up as well, we now use a decode mode enumerant which will make it easier to add in more decoders for the pak files in the future. --- r5dev/common/callback.cpp | 6 +- r5dev/common/sdkdefs.h | 4 + r5dev/engine/cmodel_bsp.cpp | 38 +- r5dev/public/rtech/ipakfile.h | 313 ++++++--- r5dev/public/rtech/rstdlib.h | 61 +- r5dev/public/tier0/jobthread.h | 10 +- r5dev/rtech/async/asyncio.h | 2 +- r5dev/rtech/pak/pakalloc.cpp | 2 +- r5dev/rtech/pak/pakdecode.cpp | 131 ++-- r5dev/rtech/pak/pakdecode.h | 6 +- r5dev/rtech/pak/pakencode.cpp | 46 +- r5dev/rtech/pak/pakencode.h | 1 + r5dev/rtech/pak/pakparse.cpp | 1115 +++++++++++++++++++++----------- r5dev/rtech/pak/pakparse.h | 33 +- r5dev/rtech/pak/pakstate.h | 47 +- r5dev/rtech/pak/pakstream.cpp | 19 +- r5dev/rtech/pak/pakstream.h | 32 +- r5dev/rtech/pak/paktools.cpp | 35 +- r5dev/rtech/pak/paktools.h | 4 +- 19 files changed, 1223 insertions(+), 682 deletions(-) diff --git a/r5dev/common/callback.cpp b/r5dev/common/callback.cpp index fc9c920e..efb08a01 100644 --- a/r5dev/common/callback.cpp +++ b/r5dev/common/callback.cpp @@ -303,9 +303,9 @@ void Pak_ListPaks_f(const CCommand& args) uint32_t nTotalLoaded = 0; - for (int16_t i = 0, n = *g_pLoadedPakCount; i < n; ++i) + for (int16_t i = 0, n = g_pakGlobals->loadedPakCount; i < n; ++i) { - const PakLoadedInfo_t& info = g_pLoadedPakInfo[i]; + const PakLoadedInfo_t& info = g_pakGlobals->loadedPaks[i]; if (info.status == EPakStatus::PAK_STATUS_FREED) continue; @@ -335,7 +335,7 @@ void Pak_ListTypes_f(const CCommand& args) for (int8_t i = 0; i < PAK_MAX_TYPES; ++i) { - PakAssetBinding_t* type = &g_pPakGlobals->assetBindings[i]; + PakAssetBinding_t* type = &g_pakGlobals->assetBindings[i]; if (!type->description) continue; diff --git a/r5dev/common/sdkdefs.h b/r5dev/common/sdkdefs.h index c13ad911..e7bb95c6 100644 --- a/r5dev/common/sdkdefs.h +++ b/r5dev/common/sdkdefs.h @@ -22,6 +22,10 @@ typedef intptr_t intp; typedef const unsigned char* rsig_t; +// 32bit and 64bit wide boolean type +typedef int32_t b32; +typedef int64_t b64; + // signed size types typedef std::make_signed_t ssize_t; diff --git a/r5dev/engine/cmodel_bsp.cpp b/r5dev/engine/cmodel_bsp.cpp index b4b71bb2..07033291 100644 --- a/r5dev/engine/cmodel_bsp.cpp +++ b/r5dev/engine/cmodel_bsp.cpp @@ -140,7 +140,7 @@ void Mod_GetAllInstalledMaps() if (!regexMatches.empty()) { - const string match = regexMatches[1].str(); + const std::sub_match& match = regexMatches[1]; if (match.compare("frontend") == 0) continue; // Frontend contains no BSP's. @@ -152,8 +152,12 @@ void Mod_GetAllInstalledMaps() continue; // Common contains mp_lobby. } - else if (!g_InstalledMaps.HasElement(match.c_str())) - g_InstalledMaps.AddToTail(match.c_str()); + else + { + const string mapName = match.str(); + if (!g_InstalledMaps.HasElement(mapName.c_str())) + g_InstalledMaps.AddToTail(mapName.c_str()); + } } } } @@ -232,15 +236,18 @@ void Mod_QueuedPakCacheFrame() { if (*data->pakName) { - const int index = data->pakId & PAK_MAX_HANDLES_MASK; + PakLoadedInfo_t* const pakInfo = Pak_GetPakInfo(data->pakId); EPakStatus status; + // TODO: revisit this, this appears incorrect but also the way + // respawn does this. it this always supposed to be true on + // retail builds? bool keepLoaded = true; data->keepLoaded = true; - if (g_pLoadedPakInfo[index].handle == data->pakId) + if (pakInfo->handle == data->pakId) { - status = g_pLoadedPakInfo[index].status; + status = pakInfo->status; keepLoaded = data->keepLoaded; } else @@ -359,12 +366,14 @@ void Mod_QueuedPakCacheFrame() { if (!*(_BYTE*)(pMTVFTaskItem + 4)) { - if (*g_pPakHasPendingUnloadJobs || *g_pLoadedPakCount != *g_pRequestedPakCount) + JobFifoLock_s* const pakFifoLock = &g_pakGlobals->fifoLock; + + if (g_pakGlobals->hasPendingUnloadJobs || g_pakGlobals->loadedPakCount != g_pakGlobals->requestedPakCount) { - if (!JT_AcquireFifoLockOrHelp(g_pPakFifoLock) - && !JT_HelpWithJobTypes((__int64(__fastcall*)(__int64, _DWORD*, __int64, _QWORD*))g_pPakFifoLockWrapper, g_pPakFifoLock, -1i64, 0i64)) + if (!JT_AcquireFifoLockOrHelp(pakFifoLock) + && !JT_HelpWithJobTypes(g_pPakFifoLockWrapper, pakFifoLock, -1i64, 0i64)) { - JT_HelpWithJobTypesOrSleep((unsigned __int8(__fastcall*)(_QWORD))g_pPakFifoLockWrapper, g_pPakFifoLock, -1i64, 0i64, 0i64, 1); + JT_HelpWithJobTypesOrSleep(g_pPakFifoLockWrapper, pakFifoLock, -1i64, 0i64, 0i64, 1); } Mod_UnloadPendingAndPrecacheRequestedPaks(); @@ -374,10 +383,11 @@ void Mod_QueuedPakCacheFrame() if (*g_bPakFifoLockAcquired) { *g_bPakFifoLockAcquired = 0; - JT_ReleaseFifoLock(g_pPakFifoLock); + JT_ReleaseFifoLock(pakFifoLock); } } - JT_ReleaseFifoLock(g_pPakFifoLock); + + JT_ReleaseFifoLock(pakFifoLock); pMTVFTaskItem = *g_pMTVFTaskItem; } @@ -413,9 +423,9 @@ CHECK_FOR_FAILURE: if (commonData->pakId != INVALID_PAK_HANDLE) { - const int infoIndex = (commonData->pakId & PAK_MAX_HANDLES_MASK); + const PakLoadedInfo_t* const pli = Pak_GetPakInfo(commonData->pakId); - if (g_pLoadedPakInfo[infoIndex].handle != commonData->pakId || ((g_pLoadedPakInfo[infoIndex].status - 9) & 0xFFFFFFFB) != 0) + if (pli->handle != commonData->pakId || ((pli->status - 9) & 0xFFFFFFFB) != 0) { *g_pPakPrecacheJobFinished = false; return; diff --git a/r5dev/public/rtech/ipakfile.h b/r5dev/public/rtech/ipakfile.h index 3683c788..e238de27 100644 --- a/r5dev/public/rtech/ipakfile.h +++ b/r5dev/public/rtech/ipakfile.h @@ -5,7 +5,6 @@ #include "rtech/iasync.h" #include "rtech/rstdlib.h" -#include "thirdparty/zstd/zstd.h" // pak header versions #define PAK_HEADER_MAGIC (('k'<<24)+('a'<<16)+('P'<<8)+'R') @@ -13,19 +12,25 @@ // pak header flags #define PAK_HEADER_FLAGS_HAS_MODULE (1<<0) +#define PAK_HEADER_FLAGS_HAS_MODULE_EXTENDED PAK_HEADER_FLAGS_HAS_MODULE | (1<<1) + #define PAK_HEADER_FLAGS_COMPRESSED (1<<8) // use the ZStd decoder instead of the RTech one -#define PAK_HEADER_FLAGS_ZSTREAM (1<<9) +#define PAK_HEADER_FLAGS_ZSTREAM_ENCODED (1<<9) // max amount of types at runtime in which assets will be tracked #define PAK_MAX_TYPES 64 #define PAK_MAX_TYPES_MASK (PAK_MAX_TYPES-1) // max amount of global pak assets at runtime -#define PAK_MAX_ASSETS 0x40000 +#define PAK_MAX_ASSETS 0x40000 // TODO: rename to PAK_MAX_LOADED_ASSETS #define PAK_MAX_ASSETS_MASK (PAK_MAX_ASSETS-1) +// max amount of global pak assets tracked at runtime +#define PAK_MAX_TRACKED_ASSETS (PAK_MAX_ASSETS/2) +#define PAK_MAX_TRACKED_ASSETS_MASK (PAK_MAX_TRACKED_ASSETS-1) + // max amount of segments a pak file could have #define PAK_MAX_SEGMENTS 20 @@ -37,7 +42,7 @@ #define PAK_MAX_STREAMING_FILE_HANDLES_PER_SET 4 // max amount of paks that could be loaded at runtime -#define PAK_MAX_HANDLES 512 +#define PAK_MAX_HANDLES 512 // TODO: rename to PAK_MAX_LOADED_PAKS #define PAK_MAX_HANDLES_MASK (PAK_MAX_HANDLES-1) // max amount of async streaming requests that could be made per pak file, if a @@ -51,7 +56,8 @@ // size is below PAK_DECODE_IN_RING_BUFFER_SIZE, the system allocates a // buffer with the size of the compressed pak + the value of this define // the system should still use the default ring buffer input size for decoding -// as these pak files get decoded in one-go; there is buffer wrapping going on +// as these pak files get decoded in one-go; there is no input buffer wrapping +// going on #define PAK_DECODE_IN_RING_BUFFER_SMALL_SIZE 0x1000 #define PAK_DECODE_IN_RING_BUFFER_SMALL_MASK (PAK_DECODE_IN_RING_BUFFER_SMALL_SIZE-1) @@ -64,6 +70,9 @@ #define PAK_DECODE_OUT_RING_BUFFER_SIZE 0x400000 #define PAK_DECODE_OUT_RING_BUFFER_MASK (PAK_DECODE_OUT_RING_BUFFER_SIZE-1) +// max amount to read per async fs read request +#define PAK_READ_DATA_CHUNK_SIZE (1ull << 19) + // base pak directory containing paks sorted in platform specific subdirectories #define PAK_BASE_PATH "paks\\" #define PLATFORM_PAK_PATH PAK_BASE_PATH"Win64\\" @@ -73,7 +82,10 @@ #define PLATFORM_PAK_OVERRIDE_PATH PAK_BASE_PATH"Win64_override\\" // the handle that should be returned when a pak failed to load or process -#define INVALID_PAK_HANDLE -1 +#define INVALID_PAK_HANDLE -1 // TODO: rename to PAK_INVALID_HANDLE + +#define PAK_MAX_DISPATCH_LOAD_JOBS 4 +#define PAK_DEFAULT_JOB_GROUP_ID 0x3000 //----------------------------------------------------------------------------- // Forward declarations @@ -116,6 +128,24 @@ union PakPage_t //----------------------------------------------------------------------------- // Enumerations //----------------------------------------------------------------------------- +enum EPakDecodeMode : int32_t +{ + MODE_DISABLED = -1, + + // the default decoder + MODE_RTECH, + MODE_ZSTD +}; + +enum EPakStreamSet +{ + STREAMING_SET_MANDATORY = 0, + STREAMING_SET_OPTIONAL, + + // number of streaming sets + STREAMING_SET_COUNT +}; + enum EPakStatus : int32_t { PAK_STATUS_FREED = 0, @@ -136,15 +166,6 @@ enum EPakStatus : int32_t PAK_STATUS_BUSY = 15 }; -enum EPakStreamSet -{ - STREAMING_SET_MANDATORY = 0, - STREAMING_SET_OPTIONAL, - - // number of streaming sets - STREAMING_SET_COUNT -}; - struct PakAssetBinding_t { enum EType @@ -221,17 +242,152 @@ struct PakAsset_t struct PakAssetShort_t { PakGuid_t guid; - uint32_t unk_8; + uint32_t trackerIndex; uint32_t unk_C; void* head; void* cpu; }; -struct PakGlobals_t +struct PakAssetTracker_s { - PakAssetBinding_t assetBindings[PAK_MAX_TYPES]; // [ PIXIE ]: Max possible registered assets on Season 3, 0-2 I did not check yet. - PakAssetShort_t assets[PAK_MAX_ASSETS]; - // end size unknown, but there appears to be stuff below too + void* memPage; + int trackerIndex; + int loadedPakIndex; + uint8_t assetTypeHashIdx; + char unk_11[3]; + char unk_10[4]; +}; + +struct PakTracker_s +{ + uint32_t numPaksTracked; + int unk_4; + int unk_8; + char gap_C[644100]; + int loadedAssetIndices[PAK_MAX_HANDLES]; + char gap_9DC04[522240]; +}; + +class PakLoadedInfo_t +{ +public: + struct StreamingInfo_t + { + inline void Reset() + { + embeddedStarpakName = nullptr; + streamFileNumber[0] = FS_ASYNC_FILE_INVALID; + streamFileNumber[1] = FS_ASYNC_FILE_INVALID; + streamFileNumber[2] = FS_ASYNC_FILE_INVALID; + streamFileNumber[3] = FS_ASYNC_FILE_INVALID; + streamFileCount = NULL; + streamingDisabled = true; + } + + char* embeddedStarpakName; + int streamFileNumber[PAK_MAX_STREAMING_FILE_HANDLES_PER_SET]; + int streamFileCount; + bool streamingDisabled; + char padding_maybe[3]; + }; + + PakHandle_t handle; + EPakStatus status; + JobID_t loadJobId; + uint32_t padding_maybe; + + // the log level of the pak, this is also used for errors if a pak failed + // to load; the higher the level, the more important this pak file is + int logLevel; + + uint32_t assetCount; + const char* fileName; + CAlignedMemAlloc* allocator; + PakGuid_t* assetGuids; //size of the array is m_nAssetCount + void* segmentBuffers[PAK_SEGMENT_BUFFER_TYPES]; + _QWORD qword50; + FILETIME fileTime; + PakFile_t* pakFile; + StreamingInfo_t streamInfo[STREAMING_SET_COUNT]; + uint32_t fileHandle; + uint8_t m_nUnk5; + HMODULE hModule; + +}; //Size: 0x00B8/0x00E8 + +struct PakGlobals_s +{ + // [ PIXIE ]: Max possible registered assets on Season 3, 0-2 I did not check yet. + PakAssetBinding_t assetBindings[PAK_MAX_TYPES]; + PakAssetShort_t loadedAssets[PAK_MAX_ASSETS]; + + // assets that are tracked across all asset types + PakAssetTracker_s trackedAssets[PAK_MAX_TRACKED_ASSETS]; + + RHashMap trackedAssetMap; // links to 'trackedAssets' + RHashMap loadedPakMap; // links to 'loadedPaks' + + // all currently loaded pak handles + PakLoadedInfo_t loadedPaks[PAK_MAX_HANDLES]; + + RMultiHashMap unkMap2; // links to 'unkIntArray' and 'unkIntArray2' + int unkIntArray[PAK_MAX_TRACKED_ASSETS]; + int unkIntArray2[PAK_MAX_ASSETS]; + + // whether asset streaming (mandatory & optional) is enabled + b64 useStreamingSystem; + + // whether we should emulate (fake) our streaming install for debugging + b64 emulateStreamingInstallInit; + b64 emulateStreamingInstall; + + // mounted # streamable assets (globally across all paks) + int64_t numStreamableAssets; + b64 hasPendingUnloadJobs; + + // paks that contain tracked assets + PakTracker_s* pakTracker; + + // called when threads have to be synced (e.g. syncing the render thread + // when we execute the unloading of paks and assets) + void* threadSyncFunc; + + // the index to the asset in the trackedAssets array of the last asset we + // tracked + int lastAssetTrackerIndex; + bool updateSplitScreenAnims; + + // the current # pending asset loading jobs + int16_t numAssetLoadJobs; + JobFifoLock_s fifoLock; + JobID_t pakLoadJobId; + + int16_t loadedPakCount; + int16_t requestedPakCount; + + PakHandle_t loadedPakHandles[PAK_MAX_HANDLES]; + + JobTypeID_t assetBindJobTypes[PAK_MAX_TYPES]; + JobTypeID_t jobTypeSlots_Unused[PAK_MAX_TYPES]; + + JobTypeID_t dispatchLoadJobTypes[PAK_MAX_DISPATCH_LOAD_JOBS]; + uint8_t dispathLoadJobPriorities[PAK_MAX_DISPATCH_LOAD_JOBS]; // promoted to JobPriority_e + uint32_t dispatchLoadJobAffinities[PAK_MAX_DISPATCH_LOAD_JOBS]; + + RTL_SRWLOCK cpuDataLock; + char unknown_or_unused[32]; + void* addToMapFunc; + void* removeFromMapFunc; + __int64 qword_167ED8540; + int dword_167ED8548; + int dword_167ED854C; + __int64 qword_167ED8550; + int dword_167ED8558; + int unknown_dword_or_nothing; + int dword_167ED8560; + int numPatchedPaks; + const char** patchedPakFiles; + uint8_t* patchNumbers; }; struct PakPatchFileHeader_t @@ -275,9 +431,19 @@ struct PakFileHeader_t return headerSize; } - inline bool IsCompressed() const + inline EPakDecodeMode GetCompressionMode() const { - return flags & PAK_HEADER_FLAGS_COMPRESSED; + if (flags & PAK_HEADER_FLAGS_ZSTREAM_ENCODED) + return EPakDecodeMode::MODE_ZSTD; + + // NOTE: this should be the first check once we rebuilt function + // 14043F300 and alloc ring buffer for the flags individually instead + // instead of having to define the main compress flag (which really + // just means that the pak is using the RTech decoder) + else if (flags & PAK_HEADER_FLAGS_COMPRESSED) + return EPakDecodeMode::MODE_RTECH; + + return EPakDecodeMode::MODE_DISABLED; } // file versions @@ -369,29 +535,45 @@ struct PakDecoder_t uint64_t outputInvMask; uint32_t headerOffset; - uint32_t padding; // unused data, available for other stuff + // this field was unused, it now contains the decoder mode + EPakDecodeMode decodeMode; + + // NOTE: unless you are in the RTech decoder, use the getter if you need to + // get the current pos!!! uint64_t inBufBytePos; + // NOTE: unless you are in the RTech decoder, use the getter if you need to + // get the current pos!!! uint64_t outBufBytePos; size_t bufferSizeNeeded; - // current byte and current bit of byte - uint64_t currentByte; - uint32_t currentBit; + union + { + // current byte and current bit of byte + uint64_t currentByte; + }; + uint32_t currentBit; uint32_t dword6C; + uint64_t qword70; union { size_t compressedStreamSize; + + // compressedStreamSize isn't used on ZStd paks, instead, we need to + // store the frame header size size_t frameHeaderSize; }; union { size_t decompressedStreamSize; + + // decompressedStreamSize isn't used on ZStd paks; use this space for + // the decoder ZSTD_DStream* zstreamContext; }; }; @@ -402,54 +584,6 @@ struct PakRingBufferFrame_t size_t frameLen; }; -struct PakFile_t; - -class PakLoadedInfo_t -{ -public: - struct StreamingInfo_t - { - inline void Reset() - { - embeddedStarpakName = nullptr; - streamFileNumber[0] = FS_ASYNC_FILE_INVALID; - streamFileNumber[1] = FS_ASYNC_FILE_INVALID; - streamFileNumber[2] = FS_ASYNC_FILE_INVALID; - streamFileNumber[3] = FS_ASYNC_FILE_INVALID; - streamFileCount = NULL; - streamingDisabled = true; - } - - char* embeddedStarpakName; - int streamFileNumber[PAK_MAX_STREAMING_FILE_HANDLES_PER_SET]; - int streamFileCount; - bool streamingDisabled; - char padding_maybe[3]; - }; - - PakHandle_t handle; - EPakStatus status; - uint64_t m_nUnk1; - - // the log level of the pak, this is also used for errors if a pak failed - // to load; the higher the level, the more important this pak file is - int logLevel; - - uint32_t assetCount; - const char* fileName; - CAlignedMemAlloc* allocator; - PakGuid_t* assetGuids; //size of the array is m_nAssetCount - void* segmentBuffers[PAK_SEGMENT_BUFFER_TYPES]; - _QWORD qword50; - FILETIME fileTime; - PakFile_t* pakFile; - StreamingInfo_t streamInfo[STREAMING_SET_COUNT]; - uint32_t fileHandle; - uint8_t m_nUnk5; - uint64_t hModule; - -}; //Size: 0x00B8/0x00E8 - struct PakFileStream_t { struct Descriptor @@ -460,20 +594,20 @@ struct PakFileStream_t // NOTE: if this is set, the game sets 'PakMemoryData_t::processedPatchedDataSize' // to 'dataOffset'; else its getting set to 'sizeof(PakFileHeader_t)'. - bool isCompressed; + EPakDecodeMode compressionMode; }; _QWORD qword0; - _QWORD qword8; + _QWORD fileSize; int fileHandle; - int gap14[32]; + int asyncRequestHandles[32]; _BYTE gap94[32]; - unsigned int unsigned_intB4; - _DWORD dwordB8; - _BYTE byteBC; + unsigned int numDataChunksProcessed; + _DWORD numDataChunks; + _BYTE fileReadStatus; bool finishedLoadingPatches; _BYTE gapBE; - _BYTE byteBF; + _BYTE numLoadedFiles; Descriptor descriptors[PAK_MAX_ASYNC_STREAMED_LOAD_REQUESTS]; uint8_t* buffer; _QWORD bufferMask; @@ -536,8 +670,8 @@ struct PakMemoryData_t uint64_t qword2D0; PakHandle_t pakId; - JobID_t unkJobID; - int* qword2E0; + JobID_t assetLoadJobId; + int* loadedAssetIndices; uint8_t** memPageBuffers; PakPatchFileHeader_t* patchHeaders; @@ -684,22 +818,13 @@ struct PakFile_t } }; -struct UnknownPakStruct_t -{ - uint32_t unk_0; - int unk_4; - int unk_8; - char gap_C[0x9D404]; - int unk_array_9D410[512]; - char gap_9DC04[0x7F800]; -}; - -static_assert(sizeof(UnknownPakStruct_t) == 0x11D410); +static_assert(sizeof(PakTracker_s) == 0x11D410); static_assert(sizeof(PakFile_t) == 2224); // S3+ static_assert(sizeof(PakLoadedInfo_t) == 184); static_assert(sizeof(PakDecoder_t) == 136); static_assert(sizeof(PakPatchFileHeader_t) == 16); static_assert(sizeof(PakPatchDataHeader_t) == 8); +static_assert(sizeof(PakGlobals_s) == 0xC98A48); #endif // RTECH_IPACKFILE_H diff --git a/r5dev/public/rtech/rstdlib.h b/r5dev/public/rtech/rstdlib.h index 594a526b..5de507fc 100644 --- a/r5dev/public/rtech/rstdlib.h +++ b/r5dev/public/rtech/rstdlib.h @@ -55,25 +55,64 @@ private: SRWLOCK m_lock; }; -#pragma pack(push, 4) -class RBitRead +struct RMultiHashMap { -public: - FORCEINLINE uint64_t ReadBits(const uint32_t numBits) + int globalIndex; + int firstBufSize; + void* firstBuffer; + void* secondBuffer; + int firstSlotsUsed; + int secondSlotsUsed; + int firstIndex; + int secondIndex; + int secondBufMask; +}; + +#pragma pack(push, 4) + +struct RBitRead +{ + unsigned __int64 m_dataBuf; + unsigned int m_bitsAvailable; + + RBitRead() : m_dataBuf(0), m_bitsAvailable(64) {}; + + FORCEINLINE void ConsumeData(unsigned __int64 input, unsigned int numBits = 64) { - Assert(numBits <= 64, "RBitRead::ReadBits: numBits must be less than or equal to 64."); + if (numBits > m_bitsAvailable) + { + assert(false && "RBitRead::ConsumeData: numBits must be less than or equal to m_bitsAvailable."); + return; + } + + m_dataBuf |= input << (64 - numBits); + } + + FORCEINLINE void ConsumeData(void* input, unsigned int numBits = 64) + { + if (numBits > m_bitsAvailable) + { + assert(false && "RBitRead::ConsumeData: numBits must be less than or equal to m_bitsAvailable."); + return; + } + + m_dataBuf |= *reinterpret_cast(input) << (64 - numBits); + } + + FORCEINLINE int BitsAvailable() const { return m_bitsAvailable; }; + + FORCEINLINE unsigned __int64 ReadBits(unsigned int numBits) + { + assert(numBits <= 64 && "RBitRead::ReadBits: numBits must be less than or equal to 64."); return m_dataBuf & ((1ull << numBits) - 1); } - FORCEINLINE void DiscardBits(const uint32_t numBits) + FORCEINLINE void DiscardBits(unsigned int numBits) { - Assert(numBits <= 64, "RBitRead::DiscardBits: numBits must be less than or equal to 64."); + assert(numBits <= 64 && "RBitRead::DiscardBits: numBits must be less than or equal to 64."); this->m_dataBuf >>= numBits; - this->m_bitsRemaining += numBits; + this->m_bitsAvailable += numBits; } - - uint64_t m_dataBuf; - int m_bitsRemaining; }; #pragma pack(pop) diff --git a/r5dev/public/tier0/jobthread.h b/r5dev/public/tier0/jobthread.h index 591061b5..744ac7db 100644 --- a/r5dev/public/tier0/jobthread.h +++ b/r5dev/public/tier0/jobthread.h @@ -4,8 +4,13 @@ typedef uint32_t JobID_t; typedef uint8_t JobTypeID_t; +typedef bool(*JobHelpCallback_t)(__int64, _DWORD*, __int64, _QWORD*); + struct JobFifoLock_s { + int id; + int depth; + short tls[64]; }; struct JobContext_s @@ -35,13 +40,14 @@ extern JobID_t JTGuts_AddJob(JobTypeID_t jobTypeId, JobID_t jobId, void* callbac inline void(*JT_ParallelCall)(void); inline void*(*JT_HelpWithAnything)(bool bShouldLoadPak); -inline bool(*JT_HelpWithJobTypes)(__int64(__fastcall* callback)(__int64, _DWORD*, __int64, _QWORD*), JobFifoLock_s* pFifoLock, __int64 a3, __int64 a4); -inline __int64(*JT_HelpWithJobTypesOrSleep)(unsigned __int8(__fastcall* a1)(_QWORD), JobFifoLock_s* pFifoLock, __int64 a3, __int64 a4, volatile signed __int64* a5, char a6); +inline bool(*JT_HelpWithJobTypes)(JobHelpCallback_t, JobFifoLock_s* pFifoLock, __int64 a3, __int64 a4); +inline __int64(*JT_HelpWithJobTypesOrSleep)(JobHelpCallback_t, JobFifoLock_s* pFifoLock, __int64 a3, __int64 a4, volatile signed __int64* a5, char a6); inline bool(*JT_AcquireFifoLockOrHelp)(struct JobFifoLock_s* pFifo); inline void(*JT_ReleaseFifoLock)(struct JobFifoLock_s* pFifo); inline void(*JT_EndJobGroup)(const JobID_t jobId); + inline unsigned int (*JT_AllocateJob)(); // Returns an index to the 'job_JT_Context' array inline JobID_t(*JTGuts_AddJob_Internal)(JobTypeID_t jobTypeId, JobID_t jobId, void* callbackfunc, void* callbackArg, int jobIndex, JobContext_s* context); diff --git a/r5dev/rtech/async/asyncio.h b/r5dev/rtech/async/asyncio.h index b5de3ea4..5bc29d74 100644 --- a/r5dev/rtech/async/asyncio.h +++ b/r5dev/rtech/async/asyncio.h @@ -53,7 +53,7 @@ extern void FS_CloseAsyncFile(const int fileHandle); inline int(*v_FS_OpenAsyncFile)(const char* const filePath, const int logLevel, size_t* const outFileSize); inline void(*v_FS_CloseAsyncFile)(const int fileHandle); -inline int(*v_FS_ReadAsyncFile)(int a1, __int64 a2, unsigned __int64 a3, void* a4, void* a5, void* a6, int a7); +inline int(*v_FS_ReadAsyncFile)(const int fileHandle, __int64 readOffset, unsigned __int64 readSize, void* a4, void* a5, void* a6, int a7); inline uint8_t(*v_FS_CheckAsyncRequest)(AsyncHandleStatus_t* pakStatus, size_t* bytesProcessed, const char** stateString); inline AsyncHandleTracker_t* g_pAsyncFileSlots; // bufSize=1024*sizeof(FileHandleTracker_t). diff --git a/r5dev/rtech/pak/pakalloc.cpp b/r5dev/rtech/pak/pakalloc.cpp index f5db45bb..b3fb82b1 100644 --- a/r5dev/rtech/pak/pakalloc.cpp +++ b/r5dev/rtech/pak/pakalloc.cpp @@ -17,7 +17,7 @@ void Pak_AlignSegmentHeaders(PakFile_t* const pak, PakSegmentDescriptor_t* const for (uint8_t i = 0; i < PAK_MAX_TYPES; ++i) { - const PakAssetBinding_t& binding = g_pPakGlobals->assetBindings[i]; + const PakAssetBinding_t& binding = g_pakGlobals->assetBindings[i]; if (desc->assetTypeCount[i]) { diff --git a/r5dev/rtech/pak/pakdecode.cpp b/r5dev/rtech/pak/pakdecode.cpp index 4600005c..cbbf229c 100644 --- a/r5dev/rtech/pak/pakdecode.cpp +++ b/r5dev/rtech/pak/pakdecode.cpp @@ -6,9 +6,6 @@ #include "tier0/binstream.h" #include "tier1/fmtstr.h" -#include "thirdparty/zstd/zstd.h" -#include "thirdparty/zstd/decompress/zstd_decompress_internal.h" - #include "rtech/ipakfile.h" #include "paktools.h" @@ -140,6 +137,8 @@ static const unsigned char /*141313180*/ s_defaultDecoderLUT[] = //----------------------------------------------------------------------------- bool Pak_HasEnoughDecodeBufferAvailable(PakDecoder_t* const decoder, const size_t outLen) { + // make sure caller has copied all data out the ring buffer first before + // overwriting it with new decoded data const uint64_t bytesWritten = (decoder->outBufBytePos & ~decoder->outputInvMask); return (outLen >= decoder->outputInvMask + (bytesWritten +1) || outLen >= decoder->decompSize); } @@ -149,8 +148,11 @@ bool Pak_HasEnoughDecodeBufferAvailable(PakDecoder_t* const decoder, const size_ //----------------------------------------------------------------------------- bool Pak_HasEnoughStreamedDataForDecode(PakDecoder_t* const decoder, const size_t inLen) { - // the decoder needs at least this many bytes to decode the current block - // buffer contiguously + // the decoder needs at least this amount of input data streamed in order + // to decode the rest of the pak file, as this is where reading has stopped + // this value may equal the currently streamed input size, as its possible + // this function is getting called to flush the remainder decoded data into + // the out buffer which got truncated off on the call prior due to wrapping return (inLen >= decoder->bufferSizeNeeded); } @@ -176,7 +178,7 @@ PakRingBufferFrame_t Pak_DetermineRingBufferFrame(const uint64_t bufMask, const //----------------------------------------------------------------------------- // initializes the RTech decoder //----------------------------------------------------------------------------- -size_t Pak_RStreamDecoderInit(PakDecoder_t* const decoder, const uint8_t* const fileBuffer, +size_t Pak_RTechDecoderInit(PakDecoder_t* const decoder, const uint8_t* const fileBuffer, const uint64_t inputMask, const size_t dataSize, const size_t dataOffset, const size_t headerSize) { uint64_t frameHeader = *(_QWORD*)((inputMask & (dataOffset + headerSize)) + fileBuffer); @@ -241,7 +243,7 @@ size_t Pak_RStreamDecoderInit(PakDecoder_t* const decoder, const uint8_t* const //----------------------------------------------------------------------------- // decodes the RTech data stream up to available buffer or data //----------------------------------------------------------------------------- -bool Pak_RStreamDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen) +bool Pak_RTechStreamDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen) { bool result; // al uint64_t outBufBytePos; // r15 @@ -568,15 +570,9 @@ LABEL_69: //----------------------------------------------------------------------------- // initializes the ZStd decoder //----------------------------------------------------------------------------- -size_t Pak_ZStreamDecoderInit(PakDecoder_t* const decoder, const uint8_t* const fileBuffer, - const uint64_t inputMask, const size_t dataSize, const size_t dataOffset, const size_t headerSize) +size_t Pak_ZStdDecoderInit(PakDecoder_t* const decoder, const uint8_t* frameHeader, + const size_t dataSize, const size_t headerSize) { - // NOTE: on original paks, this data is parsed out of the frame header, - // but for ZStd encoded paks we are always limiting this to the ring - // buffer size - decoder->inputInvMask = PAK_DECODE_OUT_RING_BUFFER_MASK; - decoder->outputInvMask = PAK_DECODE_OUT_RING_BUFFER_MASK; - ZSTD_DStream* const dctx = ZSTD_createDStream(); assert(dctx); @@ -586,16 +582,7 @@ size_t Pak_ZStreamDecoderInit(PakDecoder_t* const decoder, const uint8_t* const decoder->zstreamContext = dctx; - // this points to the first byte of the frame header, takes dataOffset - // into account which is the offset in the ring buffer to the patched - // data as we parse it contiguously after the base pak data, which - // might have ended somewhere in the middle of the ring buffer - const uint8_t* const frameHeaderData = (inputMask & (dataOffset + headerSize)) + fileBuffer; - - // this is the offset to the ZStd header in the input buffer - decoder->headerOffset = static_cast(frameHeaderData - fileBuffer); - - if (ZSTD_getFrameHeader(&dctx->fParams, frameHeaderData, dataSize) != 0) + if (ZSTD_getFrameHeader(&dctx->fParams, frameHeader, dataSize) != 0) { ZSTD_freeDStream(decoder->zstreamContext); decoder->zstreamContext = nullptr; @@ -619,51 +606,46 @@ size_t Pak_ZStreamDecoderInit(PakDecoder_t* const decoder, const uint8_t* const //----------------------------------------------------------------------------- // decodes the ZStd data stream up to available buffer or data, whichever ends -// first (determined by Pak_DetermineRingBufferFrame()) +// first //----------------------------------------------------------------------------- -bool Pak_ZStreamDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen) +bool Pak_ZStdStreamDecode(PakDecoder_t* const decoder, const PakRingBufferFrame_t& outFrame, const PakRingBufferFrame_t& inFrame) { - // must have a ZStd decoder at this point, and input seek pos may not exceed - // inLen as we can't read past currently streamed data; this should've been - // checked before calling this function - assert(decoder->zstreamContext && decoder->inBufBytePos <= inLen); - - const PakRingBufferFrame_t outFrame = Pak_DetermineRingBufferFrame(decoder->outputMask, decoder->outBufBytePos, outLen); - ZSTD_outBuffer outBuffer = { &decoder->outputBuf[outFrame.bufIndex], - outFrame.frameLen, - NULL + outFrame.frameLen, NULL }; - const PakRingBufferFrame_t inFrame = Pak_DetermineRingBufferFrame(decoder->inputMask, decoder->inBufBytePos, inLen); - ZSTD_inBuffer inBuffer = { &decoder->inputBuf[inFrame.bufIndex], - inFrame.frameLen, - NULL + inFrame.frameLen, NULL }; const size_t ret = ZSTD_decompressStream(decoder->zstreamContext, &outBuffer, &inBuffer); if (ZSTD_isError(ret)) { - DevWarning(eDLL_T::RTECH, "%s: decode error: %s\n", __FUNCTION__, ZSTD_getErrorName(ret)); + // NOTE: obtained here and not in the error formatter as we could check + // the error string during the assertion + const char* const decodeError = ZSTD_getErrorName(ret); assert(0); + Error(eDLL_T::RTECH, EXIT_FAILURE, "%s: decode error: %s\n", __FUNCTION__, decodeError); return false; } // advance buffer io positions, required so the main parser could already // start parsing the headers while the rest is getting decoded still - decoder->inBufBytePos += inBuffer.pos; decoder->outBufBytePos += outBuffer.pos; + decoder->inBufBytePos += inBuffer.pos; // on the next call, we need at least this amount of data streamed in order // to decode the rest of the pak file, as this is where reading has stopped // this value may equal the currently streamed input size, as its possible // this function is getting called to flush the remainder decoded data into // the out buffer which got truncated off on the call prior due to wrapping + // + // if the input stream has fully decoded, this should equal the size of the + // encoded pak file decoder->bufferSizeNeeded = decoder->inBufBytePos; const bool decoded = ret == NULL; @@ -683,10 +665,10 @@ bool Pak_ZStreamDecode(PakDecoder_t* const decoder, const size_t inLen, const si //----------------------------------------------------------------------------- size_t Pak_InitDecoder(PakDecoder_t* const decoder, const uint8_t* const inputBuf, uint8_t* const outputBuf, const uint64_t inputMask, const uint64_t outputMask, const size_t dataSize, const size_t dataOffset, - const size_t headerSize, const bool useZStream) + const size_t headerSize, const EPakDecodeMode decodeMode) { // buffer size must be power of two as we index into buffers using a bit - // mask rather than modulo, the mask provided must be bufferSize-1 + // mask rather than a modulo, the mask provided must be bufferSize-1 assert(IsPowerOfTwo(inputMask + 1)); assert(IsPowerOfTwo(outputMask + 1)); @@ -697,7 +679,7 @@ size_t Pak_InitDecoder(PakDecoder_t* const decoder, const uint8_t* const inputBu // the actual file size, which consists of dataOffset (anything up to the // frame header, like the file header) and the actual encoded data itself decoder->fileSize = dataOffset + dataSize; - decoder->padding = NULL; + decoder->decodeMode = decodeMode; // buffer masks, which essentially gets used to index into the input and // output buffers, similar to 'idx % bufSize', where bufSize = bufMask+1 @@ -712,10 +694,26 @@ size_t Pak_InitDecoder(PakDecoder_t* const decoder, const uint8_t* const inputBu decoder->inBufBytePos = dataOffset + headerSize; decoder->outBufBytePos = headerSize; - if (useZStream) - return Pak_ZStreamDecoderInit(decoder, inputBuf, inputMask, dataSize, dataOffset, headerSize); + // if we use the default RTech decoder, return from here as the stuff below + // is handled by the RTech decoder internally + if (decodeMode == EPakDecodeMode::MODE_RTECH) + return Pak_RTechDecoderInit(decoder, inputBuf, inputMask, dataSize, dataOffset, headerSize); - return Pak_RStreamDecoderInit(decoder, inputBuf, inputMask, dataSize, dataOffset, headerSize); + // NOTE: on RTech encoded paks this data is parsed out of the frame header, + // but for ZStd encoded paks we are always limiting this to the ring buffer + // size + decoder->outputInvMask = PAK_DECODE_OUT_RING_BUFFER_MASK; + + // this points to the first byte of the frame header, takes dataOffset + // into account which is the offset in the ring buffer to the patched + // data as we parse it contiguously after the base pak data, which + // might have ended somewhere in the middle of the ring buffer + const uint8_t* const frameHeaderData = &inputBuf[inputMask & (dataOffset + headerSize)]; + + const size_t decodeSize = Pak_ZStdDecoderInit(decoder, frameHeaderData, dataSize, headerSize); + assert(decodeSize); + + return decodeSize; } //----------------------------------------------------------------------------- @@ -723,7 +721,7 @@ size_t Pak_InitDecoder(PakDecoder_t* const decoder, const uint8_t* const inputBu // from where the base pak had ended as patch pak files are considered part of // the pak file that's currently getting loaded //----------------------------------------------------------------------------- -bool Pak_StreamToBufferDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen, const bool useZStream) +bool Pak_StreamToBufferDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen, const EPakDecodeMode decodeMode) { if (!Pak_HasEnoughStreamedDataForDecode(decoder, inLen)) return false; @@ -731,22 +729,33 @@ bool Pak_StreamToBufferDecode(PakDecoder_t* const decoder, const size_t inLen, c if (!Pak_HasEnoughDecodeBufferAvailable(decoder, outLen)) return false; - if (useZStream) - return Pak_ZStreamDecode(decoder, inLen, outLen); + if (decodeMode == EPakDecodeMode::MODE_RTECH) + return Pak_RTechStreamDecode(decoder, inLen, outLen); - return Pak_RStreamDecode(decoder, inLen, outLen); + // must have a decoder at this point + // + // also, input seek pos may not exceed inLen as we can't read past + // currently streamed data; this should've been checked before reaching + // this position in code + assert(decoder->zstreamContext && decoder->inBufBytePos <= inLen); + + const PakRingBufferFrame_t outFrame = Pak_DetermineRingBufferFrame(decoder->outputMask, decoder->outBufBytePos , outLen); + const PakRingBufferFrame_t inFrame = Pak_DetermineRingBufferFrame(decoder->inputMask, decoder->inBufBytePos, inLen); + + return Pak_ZStdStreamDecode(decoder, outFrame, inFrame); } //----------------------------------------------------------------------------- // decodes buffered input pak data //----------------------------------------------------------------------------- -bool Pak_BufferToBufferDecode(uint8_t* const inBuf, uint8_t* const outBuf, const size_t pakSize) +bool Pak_BufferToBufferDecode(uint8_t* const inBuf, uint8_t* const outBuf, const size_t pakSize, const EPakDecodeMode decodeMode) { - PakFileHeader_t* const inHeader = reinterpret_cast(inBuf); - const bool usesZStream = inHeader->flags & PAK_HEADER_FLAGS_ZSTREAM; + assert(decodeMode != EPakDecodeMode::MODE_DISABLED); PakDecoder_t decoder{}; - const size_t decompressedSize = Pak_InitDecoder(&decoder, inBuf, outBuf, UINT64_MAX, UINT64_MAX, pakSize, NULL, sizeof(PakFileHeader_t), usesZStream); + const size_t decompressedSize = Pak_InitDecoder(&decoder, inBuf, outBuf, UINT64_MAX, UINT64_MAX, pakSize, NULL, sizeof(PakFileHeader_t), decodeMode); + + PakFileHeader_t* const inHeader = reinterpret_cast(inBuf); if (decompressedSize != inHeader->decompressedSize) { @@ -757,7 +766,7 @@ bool Pak_BufferToBufferDecode(uint8_t* const inBuf, uint8_t* const outBuf, const } // we should always have enough buffer room at this point - if (!Pak_StreamToBufferDecode(&decoder, inHeader->compressedSize, inHeader->decompressedSize, usesZStream)) + if (!Pak_StreamToBufferDecode(&decoder, inHeader->compressedSize, inHeader->decompressedSize, decodeMode)) { Error(eDLL_T::RTECH, NO_ERROR, "%s: decompression failed!\n", __FUNCTION__); @@ -772,7 +781,7 @@ bool Pak_BufferToBufferDecode(uint8_t* const inBuf, uint8_t* const outBuf, const // remove compress flags outHeader->flags &= ~PAK_HEADER_FLAGS_COMPRESSED; - outHeader->flags &= ~PAK_HEADER_FLAGS_ZSTREAM; + outHeader->flags &= ~PAK_HEADER_FLAGS_ZSTREAM_ENCODED; // equal compressed size with decompressed outHeader->compressedSize = outHeader->decompressedSize; @@ -841,7 +850,9 @@ bool Pak_DecodePakFile(const char* const inPakFile, const char* const outPakFile return false; } - if (!inHeader->IsCompressed()) + const EPakDecodeMode decodeMode = inHeader->GetCompressionMode(); + + if (decodeMode == EPakDecodeMode::MODE_DISABLED) { Error(eDLL_T::RTECH, NO_ERROR, "%s: pak '%s' is already decompressed!\n", __FUNCTION__, inPakFile); @@ -862,7 +873,7 @@ bool Pak_DecodePakFile(const char* const inPakFile, const char* const outPakFile std::unique_ptr outPakBufContainer(new uint8_t[inHeader->decompressedSize]); uint8_t* const outPakBuf = outPakBufContainer.get(); - if (!Pak_BufferToBufferDecode(inPakBuf, outPakBuf, fileSize)) + if (!Pak_BufferToBufferDecode(inPakBuf, outPakBuf, fileSize, decodeMode)) { Error(eDLL_T::RTECH, NO_ERROR, "%s: failed to decompress pak file '%s'!\n", __FUNCTION__, inPakFile); diff --git a/r5dev/rtech/pak/pakdecode.h b/r5dev/rtech/pak/pakdecode.h index b15e229a..2008b012 100644 --- a/r5dev/rtech/pak/pakdecode.h +++ b/r5dev/rtech/pak/pakdecode.h @@ -4,10 +4,10 @@ extern size_t Pak_InitDecoder(PakDecoder_t* const decoder, const uint8_t* const inputBuf, uint8_t* const outputBuf, const uint64_t inputMask, const uint64_t outputMask, const size_t dataSize, const size_t dataOffset, - const size_t headerSize, const bool useZStream); + const size_t headerSize, const EPakDecodeMode decodeMode); -extern bool Pak_StreamToBufferDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen, const bool useCustom); -extern bool Pak_BufferToBufferDecode(uint8_t* const inBuf, uint8_t* const outBuf, const size_t pakSize); +extern bool Pak_StreamToBufferDecode(PakDecoder_t* const decoder, const size_t inLen, const size_t outLen, const EPakDecodeMode decodeMode); +extern bool Pak_BufferToBufferDecode(uint8_t* const inBuf, uint8_t* const outBuf, const size_t pakSize, const EPakDecodeMode decodeMode); extern bool Pak_DecodePakFile(const char* const inPakFile, const char* const outPakFile); diff --git a/r5dev/rtech/pak/pakencode.cpp b/r5dev/rtech/pak/pakencode.cpp index cef10b0a..99df0017 100644 --- a/r5dev/rtech/pak/pakencode.cpp +++ b/r5dev/rtech/pak/pakencode.cpp @@ -4,11 +4,26 @@ // //=============================================================================// #include "tier0/binstream.h" -#include "thirdparty/zstd/zstd.h" #include "rtech/ipakfile.h" #include "paktools.h" #include "pakencode.h" +//----------------------------------------------------------------------------- +// determines whether encoding had failed +//----------------------------------------------------------------------------- +static bool Pak_HasEncodeFailed(const size_t result) +{ + return static_cast(ZSTD_isError(result)); +} + +//----------------------------------------------------------------------------- +// returns the encode error as string +//----------------------------------------------------------------------------- +static const char* Pak_GetEncodeError(const size_t result) +{ + return ZSTD_getErrorName(result); +} + //----------------------------------------------------------------------------- // encodes the pak file from buffer, we can't do streamed compression as we // need to know the actual decompress size ahead of time, else the runtime will @@ -22,17 +37,20 @@ bool Pak_BufferToBufferEncode(const uint8_t* const inBuf, const uint64_t inLen, // compressed const size_t dataOffset = sizeof(PakFileHeader_t); - const size_t compressSize = ZSTD_compress( - outBuf + dataOffset, - outLen - dataOffset, - inBuf + dataOffset, - inLen - dataOffset, - level); + uint8_t* const dstBuf = outBuf + dataOffset; + const size_t dstLen = outLen - dataOffset; - if (ZSTD_isError(compressSize)) + const uint8_t* const srcBuf = inBuf + dataOffset; + const size_t srcLen = inLen - dataOffset; + + size_t compressSize = NULL; + + compressSize = ZSTD_compress(dstBuf, dstLen, srcBuf, srcLen, level); + + if (Pak_HasEncodeFailed(compressSize)) { Error(eDLL_T::RTECH, NO_ERROR, "%s: compression failed! [%s]\n", - __FUNCTION__, ZSTD_getErrorName(compressSize)); + __FUNCTION__, Pak_GetEncodeError(compressSize)); return false; } @@ -46,7 +64,7 @@ bool Pak_BufferToBufferEncode(const uint8_t* const inBuf, const uint64_t inLen, // these flags are required for the game's runtime to decide whether or not to // decompress the pak, and how; see Pak_ProcessPakFile() for more details outHeader->flags |= PAK_HEADER_FLAGS_COMPRESSED; - outHeader->flags |= PAK_HEADER_FLAGS_ZSTREAM; + outHeader->flags |= PAK_HEADER_FLAGS_ZSTREAM_ENCODED; return true; } @@ -111,7 +129,7 @@ bool Pak_EncodePakFile(const char* const inPakFile, const char* const outPakFile return false; } - if (inHeader->IsCompressed()) + if (inHeader->GetCompressionMode() != EPakDecodeMode::MODE_DISABLED) { Error(eDLL_T::RTECH, NO_ERROR, "%s: pak '%s' is already compressed!\n", __FUNCTION__, inPakFile); @@ -135,7 +153,9 @@ bool Pak_EncodePakFile(const char* const inPakFile, const char* const outPakFile __FUNCTION__, inPakFile); } - std::unique_ptr outPakBufContainer(new uint8_t[inHeader->decompressedSize]); + const size_t outBufSize = inHeader->decompressedSize; + + std::unique_ptr outPakBufContainer(new uint8_t[outBufSize]); uint8_t* const outPakBuf = outPakBufContainer.get(); PakFileHeader_t* const outHeader = reinterpret_cast(outPakBuf); @@ -144,7 +164,7 @@ bool Pak_EncodePakFile(const char* const inPakFile, const char* const outPakFile *outHeader = *inHeader; // encoding failed - if (!Pak_BufferToBufferEncode(inPakBuf, inHeader->compressedSize, outPakBuf, inHeader->decompressedSize, level)) + if (!Pak_BufferToBufferEncode(inPakBuf, fileSize, outPakBuf, outBufSize, level)) { Error(eDLL_T::RTECH, NO_ERROR, "%s: failed to compress pak file '%s'!\n", __FUNCTION__, inPakFile); diff --git a/r5dev/rtech/pak/pakencode.h b/r5dev/rtech/pak/pakencode.h index 4e567b89..afe54992 100644 --- a/r5dev/rtech/pak/pakencode.h +++ b/r5dev/rtech/pak/pakencode.h @@ -1,5 +1,6 @@ #ifndef RTECH_PAKENCODE_H #define RTECH_PAKENCODE_H +#include "rtech/ipakfile.h" bool Pak_BufferToBufferEncode(const uint8_t* const inBuf, const uint64_t inLen, uint8_t* const outBuf, const uint64_t outLen, const int level); diff --git a/r5dev/rtech/pak/pakparse.cpp b/r5dev/rtech/pak/pakparse.cpp index eefba6a8..2aadcdd7 100644 --- a/r5dev/rtech/pak/pakparse.cpp +++ b/r5dev/rtech/pak/pakparse.cpp @@ -34,7 +34,7 @@ static bool Pak_ResolveAssetDependency(const PakFile_t* const pak, PakGuid_t cur return false; currentIndex &= PAK_MAX_ASSETS_MASK; - currentGuid = g_pPakGlobals->assets[currentIndex].guid; + currentGuid = g_pakGlobals->loadedAssets[currentIndex].guid; if (currentGuid == targetGuid) return true; @@ -49,7 +49,7 @@ static bool Pak_ResolveAssetDependency(const PakFile_t* const pak, PakGuid_t cur void Pak_ResolveAssetRelations(PakFile_t* const pak, const PakAsset_t* const asset) { PakPage_t* const pGuidDescriptors = &pak->memoryData.guidDescriptors[asset->dependenciesIndex]; - volatile uint32_t* v5 = reinterpret_cast(*(reinterpret_cast(g_pPakGlobals) + 0x17 * (pak->memoryData.pakId & PAK_MAX_HANDLES_MASK) + 0x160212)); + uint32_t* const v5 = (uint32_t*)g_pakGlobals->loadedPaks[pak->memoryData.pakId & PAK_MAX_HANDLES_MASK].qword50; if (pak_debugrelations->GetBool()) Msg(eDLL_T::RTECH, "Resolving relations for asset: '0x%-16llX', dependencies: %-4u; in pak '%s'\n", @@ -64,7 +64,7 @@ void Pak_ResolveAssetRelations(PakFile_t* const pak, const PakAsset_t* const ass // get asset index int currentIndex = targetGuid & PAK_MAX_ASSETS_MASK; - const PakGuid_t currentGuid = g_pPakGlobals->assets[currentIndex].guid; + const PakGuid_t currentGuid = g_pakGlobals->loadedAssets[currentIndex].guid; const int64_t v9 = 2i64 * InterlockedExchangeAdd(v5, 1u); *reinterpret_cast(const_cast(&v5[2 * v9 + 2])) = targetGuid; @@ -100,12 +100,12 @@ void Pak_ResolveAssetRelations(PakFile_t* const pak, const PakAsset_t* const ass } } - currentIndex = pak->memoryData.qword2E0[a]; + currentIndex = pak->memoryData.loadedAssetIndices[a]; } } // finally write the pointer to the guid entry - *pCurrentGuid = g_pPakGlobals->assets[currentIndex].head; + *pCurrentGuid = g_pakGlobals->loadedAssets[currentIndex].head; } } @@ -131,59 +131,57 @@ uint32_t Pak_ProcessRemainingPagePointers(PakFile_t* const pak) return processedPointers; } -void __fastcall Rebuild_14043E030(PakFile_t* const pak) +void Pak_RunAssetLoadingJobs(PakFile_t* const pak) { - __int64 numAssets; // rsi - PakAsset_t* pakAsset; // rdi - __int64 _numAssets; // r14 - unsigned int assetBind; // ebp - __int64 v13; // rcx - __int64 qword2D8_low; // rdi - JobID_t qword2D8_high; // ebp - JobTypeID_t v16; // si - pak->numProcessedPointers = Pak_ProcessRemainingPagePointers(pak); - numAssets = (unsigned int)pak->processedAssetCount; - if ((_DWORD)numAssets != pak->memoryData.pakHeader.assetCount) + const uint32_t numAssets = pak->processedAssetCount; + + if (numAssets == pak->memoryData.pakHeader.assetCount) + return; + + PakAsset_t* pakAsset = &pak->memoryData.assetEntries[numAssets]; + + if (pakAsset->pageEnd > pak->processedPageCount) + return; + + for (uint32_t currentAsset = numAssets; g_pakGlobals->numAssetLoadJobs <= 0xC8u;) { - pakAsset = &pak->memoryData.assetEntries[numAssets]; - _numAssets = (unsigned int)numAssets; - if ((int)pakAsset->pageEnd <= pak->processedPageCount) + pak->memoryData.loadedAssetIndices[currentAsset] = Pak_TrackAsset(pak, pakAsset); + + _InterlockedIncrement16(&g_pakGlobals->numAssetLoadJobs); + + const uint8_t assetBind = pakAsset->HashTableIndexForAssetType(); + + if (g_pakGlobals->assetBindings[assetBind].loadAssetFunc) { - while ((unsigned __int16)*word_167ED7BDE <= 0xC8u) - { - assetBind = pakAsset->HashTableIndexForAssetType(); - pak->memoryData.qword2E0[_numAssets] = (int)sub_14043D3C0(pak, pakAsset); - _InterlockedIncrement16(word_167ED7BDE); - v13 = assetBind; - if (g_pPakGlobals->assetBindings[(unsigned __int64)assetBind].loadAssetFunc) - { - qword2D8_low = pak->memoryData.pakId; - qword2D8_high = pak->memoryData.unkJobID; - v16 = *((_BYTE*)&*g_pPakGlobals + v13 + 13207872); + const JobTypeID_t jobTypeId = g_pakGlobals->assetBindJobTypes[assetBind]; - JTGuts_AddJob(v16, qword2D8_high, (void*)qword2D8_low, (void*)_numAssets); - } - else - { - if (_InterlockedExchangeAdd16((volatile signed __int16*)&pakAsset->numRemainingDependencies, 0xFFFFu) == 1) - sub_14043D150(pak, pakAsset, (unsigned int)numAssets, assetBind); - _InterlockedDecrement16(word_167ED7BDE); - } - numAssets = (unsigned int)++pak->processedAssetCount; - if ((_DWORD)numAssets == pak->memoryData.pakHeader.assetCount) - { - JT_EndJobGroup(pak->memoryData.unkJobID); - return; - } - _numAssets = (unsigned int)numAssets; - pakAsset = &pak->memoryData.assetEntries[numAssets]; + // have to cast it to a bigger size to send it as param to JTGuts_AddJob(). + const int64_t pakId = pak->memoryData.pakId; - if (pakAsset->pageEnd > pak->processedPageCount) - return; - } + JTGuts_AddJob(jobTypeId, pak->memoryData.assetLoadJobId, (void*)pakId, (void*)(uint64_t)currentAsset); } + else + { + if (_InterlockedExchangeAdd16((volatile signed __int16*)&pakAsset->numRemainingDependencies, 0xFFFFu) == 1) + Pak_ProcessAssetRelationsAndResolveDependencies(pak, pakAsset, currentAsset, assetBind); + + _InterlockedDecrement16(&g_pakGlobals->numAssetLoadJobs); + } + + currentAsset = ++pak->processedAssetCount; + + if (currentAsset == pak->memoryData.pakHeader.assetCount) + { + JT_EndJobGroup(pak->memoryData.assetLoadJobId); + return; + } + + pakAsset = &pak->memoryData.assetEntries[currentAsset]; + + if (pakAsset->pageEnd > pak->processedPageCount) + return; } } @@ -217,7 +215,7 @@ void Pak_UnloadAsync(PakHandle_t handle) { const PakLoadedInfo_t* const pakInfo = Pak_GetPakInfo(handle); - if (pakInfo && pakInfo->fileName) + if (pakInfo->fileName) Msg(eDLL_T::RTECH, "Unloading pak file: '%s'\n", pakInfo->fileName); v_Pak_UnloadAsync(handle); @@ -234,246 +232,170 @@ static const int s_patchCmdToBytesToProcess[] = { CMD_INVALID, CMD_INVALID, CMD_ //---------------------------------------------------------------------------------- bool Pak_ProcessPakFile(PakFile_t* const pak) { - PakFileHeader_t* pakHeader; // r8 - PakFileStream_t* fileStream; // rsi - PakMemoryData_t* memoryData; // r14 - __int64 dwordB8; // rcx - uint32_t v6; // eax - __int64 v7; // rax - char v8; // r13 - uint64_t bytesProcessed; // r12 - unsigned __int64 v16; // r9 - unsigned __int8 v17; // cl - unsigned __int64 v18; // r8 - uint8_t byte1F8; // al - uint8_t byte1FD; // cl - PakFileStream_t::Descriptor* v22; // rdi - uint64_t dataOffset; // rax - PakDecoder_t* decodeContext; // rbp - uint64_t decompressedSize; // rax - uint64_t compressedSize; // rdx - uint64_t qword1D0; // rcx - __int64 v28; // rax - unsigned int numBitsRemaining; // r8d - int v35; // ecx - int v39; // r10d - int v40; // r9d - uint64_t v42; // ecx - unsigned int v43; // r8d - uint64_t v44; // r12d - char byteBC; // r15 - __int64 v46; // rbp - __int64 v47; // r8 - unsigned __int64 v48; // rbp - unsigned __int64 qword8; // rax - __int64 patchCount; // rcx - char c; // al - char* it; // rcx - char* i; // rdx - int v56; // edi - unsigned __int64 v58; // rdx - char pakPatchPath[MAX_PATH]; // [rsp+40h] [rbp-148h] BYREF - unsigned __int64 v62; // [rsp+190h] [rbp+8h] - size_t numBytesToProcess; // [rsp+198h] [rbp+10h] BYREF + PakFileStream_t* const fileStream = &pak->fileStream; + PakMemoryData_t* const memoryData = &pak->memoryData; - fileStream = &pak->fileStream; - memoryData = &pak->memoryData; - dwordB8 = (unsigned int)pak->fileStream.dwordB8; + // first request is always just the header? + size_t readStart = sizeof(PakFileHeader_t); - if ((_DWORD)dwordB8) - v62 = dwordB8 << 19; - else - v62 = sizeof(PakFileHeader_t); + if (fileStream->numDataChunks > 0) + readStart = fileStream->numDataChunks * PAK_READ_DATA_CHUNK_SIZE; - v6 = fileStream->unsigned_intB4; - if (v6 != dwordB8) + for (; fileStream->numDataChunksProcessed != fileStream->numDataChunks; fileStream->numDataChunksProcessed++) { - while (1) + const int v7 = fileStream->numDataChunksProcessed & 0x1F; + const uint8_t v8 = fileStream->gap94[v7]; + if (v8 != 1) { - v7 = v6 & 0x1F; - v8 = fileStream->gap94[v7]; - if (v8 != 1) + size_t bytesProcessed = 0; + const char* statusMsg = "(no reason)"; + const uint8_t currentStatus = g_pakLoadApi->CheckAsyncRequest(fileStream->asyncRequestHandles[v7], &bytesProcessed, &statusMsg); + + if (currentStatus == AsyncHandleStatus_t::FS_ASYNC_ERROR) + Error(eDLL_T::RTECH, EXIT_FAILURE, "Error reading pak file \"%s\" -- %s\n", pak->memoryData.fileName, statusMsg); + else if (currentStatus == AsyncHandleStatus_t::FS_ASYNC_PENDING) break; - LABEL_17: - v6 = fileStream->unsigned_intB4 + 1; - fileStream->unsigned_intB4 = v6; - if (v6 == fileStream->dwordB8) - goto LABEL_18; - } - const char* statusMsg = "(no reason)"; - const uint8_t currentStatus = g_pakLoadApi->CheckAsyncRequest((unsigned char)fileStream->gap14[v7], &bytesProcessed, &statusMsg); - - if (currentStatus == AsyncHandleStatus_t::FS_ASYNC_PENDING) - goto LABEL_18; - - if (currentStatus == AsyncHandleStatus_t::FS_ASYNC_ERROR) - Error(eDLL_T::RTECH, EXIT_FAILURE, "Error reading pak file \"%s\" -- %s\n", pak->memoryData.fileName, statusMsg); - - fileStream->bytesStreamed += bytesProcessed; - if (v8) - { - pakHeader = &pak->memoryData.pakHeader; - v16 = (unsigned __int64)fileStream->unsigned_intB4 << 19; - v17 = fileStream->byteBF++ & PAK_MAX_ASYNC_STREAMED_LOAD_REQUESTS_MASK; - - if (v8 == 2) + fileStream->bytesStreamed += bytesProcessed; + if (v8) { - v18 = v16 & fileStream->bufferMask; - fileStream->bytesStreamed = bytesProcessed + v16; - pakHeader = (PakFileHeader_t*)&fileStream->buffer[v18]; + const PakFileHeader_t* pakHeader = &pak->memoryData.pakHeader; + const uint64_t v16 = fileStream->numDataChunksProcessed * PAK_READ_DATA_CHUNK_SIZE; + + if (v8 == 2) + { + fileStream->bytesStreamed = bytesProcessed + v16; + pakHeader = (PakFileHeader_t*)&fileStream->buffer[v16 & fileStream->bufferMask]; + } + + const uint8_t fileIndex = fileStream->numLoadedFiles++ & PAK_MAX_ASYNC_STREAMED_LOAD_REQUESTS_MASK; + + //printf("v16: %lld\n", v16); + fileStream->descriptors[fileIndex].dataOffset = v16 + sizeof(PakFileHeader_t); + fileStream->descriptors[fileIndex].compressedSize = v16 + pakHeader->compressedSize; + fileStream->descriptors[fileIndex].decompressedSize = pakHeader->decompressedSize; + fileStream->descriptors[fileIndex].compressionMode = pakHeader->GetCompressionMode(); } - - fileStream->descriptors[v17].dataOffset = v16 + sizeof(PakFileHeader_t); - fileStream->descriptors[v17].compressedSize = v16 + pakHeader->compressedSize; - fileStream->descriptors[v17].decompressedSize = pakHeader->decompressedSize; - fileStream->descriptors[v17].isCompressed = pakHeader->IsCompressed(); } - goto LABEL_17; } -LABEL_18: - byte1F8 = pak->byte1F8; - if (byte1F8 != fileStream->byteBF) + + size_t qword1D0 = memoryData->processedPatchedDataSize; + + for (; pak->byte1F8 != fileStream->numLoadedFiles; pak->byte1F8++) { - byte1FD = pak->byte1FD; - const bool useZStream = pak->GetHeader().flags & PAK_HEADER_FLAGS_ZSTREAM; + PakFileStream_t::Descriptor* v22 = &fileStream->descriptors[pak->byte1F8 & PAK_MAX_ASYNC_STREAMED_LOAD_REQUESTS_MASK]; - do + if (pak->byte1FD) { - v22 = &fileStream->descriptors[byte1F8 & PAK_MAX_ASYNC_STREAMED_LOAD_REQUESTS_MASK]; + pak->byte1FD = false; + pak->inputBytePos = v22->dataOffset; - if (byte1FD) + if (v22->compressionMode != EPakDecodeMode::MODE_DISABLED) { - pak->byte1FD = 0; - pak->inputBytePos = v22->dataOffset; - - if (v22->isCompressed) - { - pak->isOffsetted_MAYBE = 0; - pak->isCompressed = 1; - dataOffset = sizeof(PakFileHeader_t); - } - else - { - pak->isOffsetted_MAYBE = 1; - pak->isCompressed = 0; - dataOffset = v22->dataOffset; - } - - memoryData->processedPatchedDataSize = dataOffset; - - if (!pak->isCompressed) - { - LABEL_35: - compressedSize = v22->compressedSize; - qword1D0 = compressedSize; - if (fileStream->bytesStreamed < compressedSize) - qword1D0 = fileStream->bytesStreamed; - goto LABEL_41; - } - - decodeContext = &pak->pakDecoder; - - decompressedSize = Pak_InitDecoder(&pak->pakDecoder, - fileStream->buffer, pak->decompBuffer, - PAK_DECODE_IN_RING_BUFFER_MASK, PAK_DECODE_OUT_RING_BUFFER_MASK, - v22->compressedSize - (v22->dataOffset - sizeof(PakFileHeader_t)), - v22->dataOffset - sizeof(PakFileHeader_t), sizeof(PakFileHeader_t), useZStream); - - if (decompressedSize != v22->decompressedSize) - Error(eDLL_T::RTECH, EXIT_FAILURE, - "Error reading pak file \"%s\" -- decompressed size %zu doesn't match expected value %zu\n", - pak->memoryData.fileName, - decompressedSize, - pak->memoryData.pakHeader.decompressedSize); + pak->isOffsetted_MAYBE = false; + pak->isCompressed = true; + memoryData->processedPatchedDataSize = sizeof(PakFileHeader_t); } else { - decodeContext = &pak->pakDecoder; + pak->isOffsetted_MAYBE = true; + pak->isCompressed = false; + //printf("v22->dataOffset: %lld\n", v22->dataOffset); + memoryData->processedPatchedDataSize = v22->dataOffset; } - if (!pak->isCompressed) - goto LABEL_35; + if (pak->isCompressed) + { + const size_t decompressedSize = Pak_InitDecoder(&pak->pakDecoder, + fileStream->buffer, pak->decompBuffer, + PAK_DECODE_IN_RING_BUFFER_MASK, PAK_DECODE_OUT_RING_BUFFER_MASK, + v22->compressedSize - (v22->dataOffset - sizeof(PakFileHeader_t)), + v22->dataOffset - sizeof(PakFileHeader_t), sizeof(PakFileHeader_t), v22->compressionMode); + if (decompressedSize != v22->decompressedSize) + Error(eDLL_T::RTECH, EXIT_FAILURE, + "Error reading pak file \"%s\" with decoder \"%s\" -- decompressed size %zu doesn't match expected value %zu\n", + pak->memoryData.fileName, + Pak_DecoderToString(v22->compressionMode), + decompressedSize, + pak->memoryData.pakHeader.decompressedSize); + } + } + + if (pak->isCompressed) + { qword1D0 = pak->pakDecoder.outBufBytePos; if (qword1D0 != pak->pakDecoder.decompSize) { - const bool didDecode = Pak_StreamToBufferDecode(decodeContext, fileStream->bytesStreamed, memoryData->processedPatchedDataSize + PAK_DECODE_OUT_RING_BUFFER_SIZE, useZStream); + const bool didDecode = Pak_StreamToBufferDecode(&pak->pakDecoder, + fileStream->bytesStreamed, (memoryData->processedPatchedDataSize + PAK_DECODE_OUT_RING_BUFFER_SIZE), v22->compressionMode); qword1D0 = pak->pakDecoder.outBufBytePos; + pak->inputBytePos = pak->pakDecoder.inBufBytePos; if (didDecode) DevMsg(eDLL_T::RTECH, "%s: pak '%s' decoded successfully\n", __FUNCTION__, pak->GetName()); } + } + else + { + qword1D0 = Min(v22->compressedSize, fileStream->bytesStreamed); + } - compressedSize = v22->compressedSize; + if (pak->inputBytePos != v22->compressedSize || memoryData->processedPatchedDataSize != qword1D0) + break; - LABEL_41: - if (pak->inputBytePos != compressedSize || memoryData->processedPatchedDataSize != qword1D0) - goto LABEL_45; - - byte1FD = 1; - byte1F8 = pak->byte1F8 + 1; - pak->byte1FD = 1; - pak->byte1F8 = byte1F8; - } while (byte1F8 != fileStream->byteBF); + pak->byte1FD = true; + qword1D0 = memoryData->processedPatchedDataSize; } - qword1D0 = memoryData->processedPatchedDataSize; -LABEL_45: - v28 = memoryData->field_2A8; - numBytesToProcess = qword1D0 - memoryData->processedPatchedDataSize; + size_t numBytesToProcess = qword1D0 - memoryData->processedPatchedDataSize; - if (memoryData->patchSrcSize + v28) + // DEBUG: REMOVE + if ((long long)pak->memoryData.patchFunc == 0x14043E2A0) + pak->memoryData.patchFunc = g_pakPatchApi[0]; + + while (memoryData->patchSrcSize + memoryData->field_2A8) { - do + // if there are no bytes left to process in this patch operation + if (!memoryData->numBytesToProcess_maybe) { - // if there are no bytes left to process in this patch operation - if (!memoryData->numBytesToProcess_maybe) + RBitRead& bitbuf = memoryData->bitBuf; + bitbuf.ConsumeData(memoryData->patchData, bitbuf.BitsAvailable()); + + // advance patch data buffer by the number of bytes that have just been fetched + memoryData->patchData = &memoryData->patchData[bitbuf.BitsAvailable() >> 3]; + + // store the number of bits remaining to complete the data read + bitbuf.m_bitsAvailable = bitbuf.BitsAvailable() & 7; // number of bits above a whole byte + + const __int8 cmd = memoryData->patchCommands[bitbuf.ReadBits(6)]; + + bitbuf.DiscardBits(memoryData->PATCH_field_68[bitbuf.ReadBits(6)]); + + // get the next patch function to execute + memoryData->patchFunc = g_pakPatchApi[cmd]; + + if (cmd <= 3u) { - RBitRead& bitbuf = memoryData->bitBuf; + const uint8_t bitExponent = memoryData->PATCH_unk2[bitbuf.ReadBits(8)]; // number of stored bits for the data size - numBitsRemaining = bitbuf.m_bitsRemaining; // number of "free" bits in the bitbuf (how many to fetch) + bitbuf.DiscardBits(memoryData->PATCH_unk3[bitbuf.ReadBits(8)]); - // fetch remaining bits - bitbuf.m_dataBuf |= *(_QWORD*)memoryData->patchData << (64 - (unsigned __int8)numBitsRemaining); + memoryData->numBytesToProcess_maybe = (1ull << bitExponent) + bitbuf.ReadBits(bitExponent); - // advance patch data buffer by the number of bytes that have just been fetched - memoryData->patchData = &memoryData->patchData[numBitsRemaining >> 3]; - - // store the number of bits remaining to complete the data read - bitbuf.m_bitsRemaining = numBitsRemaining & 7; // number of bits above a whole byte - - const unsigned __int8 index1 = static_cast(bitbuf.ReadBits(6)); - v35 = memoryData->PATCH_field_68[index1]; // number of bits to discard from bitbuf - __int8 cmd = memoryData->patchCommands[index1]; - - bitbuf.DiscardBits(v35); - - // get the next patch function to execute - memoryData->patchFunc = g_pakPatchApi[cmd]; - - if (cmd <= 3u) - { - const unsigned __int8 index2 = static_cast(bitbuf.ReadBits(8)); - v39 = memoryData->PATCH_unk3[index2]; - v40 = memoryData->PATCH_unk2[index2]; // number of stored bits for the data size - - bitbuf.DiscardBits(v39); - - memoryData->numBytesToProcess_maybe = (1ull << v40) + bitbuf.ReadBits(v40); - - bitbuf.DiscardBits(v40); - } - else - { - memoryData->numBytesToProcess_maybe = s_patchCmdToBytesToProcess[cmd]; - } + bitbuf.DiscardBits(bitExponent); } + else + { + memoryData->numBytesToProcess_maybe = s_patchCmdToBytesToProcess[cmd]; + } + } - } while (pak->memoryData.patchFunc(pak, &numBytesToProcess) && memoryData->patchSrcSize + memoryData->field_2A8); + if (!pak->memoryData.patchFunc(pak, &numBytesToProcess)) + break; } if (pak->isOffsetted_MAYBE) @@ -481,120 +403,118 @@ LABEL_45: if (!fileStream->finishedLoadingPatches) { - v42 = fileStream->unsigned_intB4; - v43 = fileStream->dwordB8; + const size_t v42 = min(fileStream->numDataChunksProcessed, pak->inputBytePos >> 19); - if ((unsigned int)(pak->inputBytePos >> 19) < v42) - v42 = pak->inputBytePos >> 19; + //if ((unsigned int)(pak->inputBytePos >> 19) < v42) + // v42 = pak->inputBytePos >> 19; - v44 = v42 + 32; - - if (v43 != v42 + 32) + while (fileStream->numDataChunks != v42 + 32) { - while (1) + const int8_t requestIdx = fileStream->numDataChunks & 0x1F; + const size_t readOffsetEnd = (fileStream->numDataChunks + 1ull) * PAK_READ_DATA_CHUNK_SIZE; + + if (fileStream->fileReadStatus == 1) { - byteBC = fileStream->byteBC; - v46 = v43; - v47 = v43 & 0x1F; - v48 = (v46 + 1) << 19; + fileStream->asyncRequestHandles[requestIdx] = FS_ASYNC_REQ_INVALID; + fileStream->gap94[requestIdx] = 1; - if (byteBC == 1) - break; + if (((requestIdx + 1) & PAK_MAX_ASYNC_STREAMED_LOAD_REQUESTS_MASK) == 0) + fileStream->fileReadStatus = 2; - qword8 = fileStream->qword8; - if (v62 < qword8) + ++fileStream->numDataChunks; + readStart = readOffsetEnd; + } + else + { + if (readStart < fileStream->fileSize) { - if (v48 < qword8) - qword8 = v48; + const size_t lenToRead = Min(fileStream->fileSize, readOffsetEnd); - fileStream->gap14[v47] = v_FS_ReadAsyncFile( + const size_t readOffset = readStart - fileStream->qword0; + const size_t readSize = lenToRead - readStart; + + fileStream->asyncRequestHandles[requestIdx] = v_FS_ReadAsyncFile( fileStream->fileHandle, - v62 - fileStream->qword0, - qword8 - v62, - &fileStream->buffer[v62 & fileStream->bufferMask], - 0i64, - 0i64, + readOffset, + readSize, + &fileStream->buffer[readStart & fileStream->bufferMask], + 0, + 0, 4); - fileStream->gap94[v47] = byteBC; - fileStream->byteBC = 0; - goto LABEL_65; + fileStream->gap94[requestIdx] = fileStream->fileReadStatus; + fileStream->fileReadStatus = 0; + + ++fileStream->numDataChunks; + readStart = readOffsetEnd; } - - if (pak->patchCount >= pak->memoryData.pakHeader.patchIndex) + else { - FS_CloseAsyncFile((short)fileStream->fileHandle); - fileStream->fileHandle = INVALID_PAK_HANDLE; - fileStream->qword0 = 0i64; - fileStream->finishedLoadingPatches = true; - - return memoryData->patchSrcSize == 0; - } - - if (!pak->dword14) - return memoryData->patchSrcSize == 0; - - sprintf(pakPatchPath, PLATFORM_PAK_PATH"%s", pak->memoryData.fileName); - patchCount = pak->patchCount++; - - // get path of next patch rpak to load - if (pak->memoryData.patchIndices[patchCount]) - { - c = pakPatchPath[0]; - it = pakPatchPath; - - for (i = nullptr; c; ++it) + if (pak->patchCount >= pak->memoryData.pakHeader.patchIndex) { - if (c == '.') - { - i = it; - } - else if (c == '\\' || c == '/') - { - i = nullptr; - } - c = it[1]; + FS_CloseAsyncFile(fileStream->fileHandle); + fileStream->fileHandle = INVALID_PAK_HANDLE; + fileStream->qword0 = 0; + fileStream->finishedLoadingPatches = true; + + return memoryData->patchSrcSize == 0; } - if (i) - it = i; - // replace extension '.rpak' with '(xx).rpak' - snprintf(it, &pakPatchPath[sizeof(pakPatchPath)] - it, - "(%02u).rpak", pak->memoryData.patchIndices[patchCount]); + if (!pak->dword14) + return memoryData->patchSrcSize == 0; + + char pakPatchPath[MAX_PATH] = {}; + sprintf(pakPatchPath, PLATFORM_PAK_PATH"%s", pak->memoryData.fileName); + + // get path of next patch rpak to load + if (pak->memoryData.patchIndices[pak->patchCount]) + { + char* pExtension = nullptr; + + char* it = pakPatchPath; + while (*it) + { + if (*it == '.') + pExtension = it; + else if (*it == '\\' || *it == '/') + pExtension = nullptr; + + ++it; + } + + if (pExtension) + it = pExtension; + + // replace extension '.rpak' with '(xx).rpak' + snprintf(it, &pakPatchPath[sizeof(pakPatchPath)] - it, + "(%02u).rpak", pak->memoryData.patchIndices[pak->patchCount]); + } + + const int patchFileHandle = FS_OpenAsyncFile(pakPatchPath, 5, &numBytesToProcess); + + //printf("[%s] Opened pak '%s' with file handle %i\n", pak->GetName(), pakPatchPath, patchFileHandle); + + if (patchFileHandle == FS_ASYNC_FILE_INVALID) + Error(eDLL_T::RTECH, EXIT_FAILURE, "Couldn't open file \"%s\".\n", pakPatchPath); + + if (numBytesToProcess < pak->memoryData.patchHeaders[pak->patchCount].compressedSize) + Error(eDLL_T::RTECH, EXIT_FAILURE, "File \"%s\" appears truncated; read size: %zu < expected size: %zu.\n", + pakPatchPath, numBytesToProcess, pak->memoryData.patchHeaders[pak->patchCount].compressedSize); + + FS_CloseAsyncFile(fileStream->fileHandle); + + fileStream->fileHandle = patchFileHandle; + + const size_t v58 = ALIGN_VALUE(fileStream->numDataChunks, 8ull) * PAK_READ_DATA_CHUNK_SIZE; + fileStream->fileReadStatus = (fileStream->numDataChunks == ALIGN_VALUE(fileStream->numDataChunks, 8ull)) + 1; + + //printf("[%s] dwordB8: %i, v58: %lld, byteBC: %i, numFiles: %i\n", pak->GetName(), fileStream->numDataChunks, v58, fileStream->byteBC, fileStream->numLoadedFiles); + fileStream->qword0 = v58; + fileStream->fileSize = v58 + pak->memoryData.patchHeaders[pak->patchCount].compressedSize; + + pak->patchCount++; } - - v56 = FS_OpenAsyncFile(pakPatchPath, 5i64, &numBytesToProcess); - - if (v56 == FS_ASYNC_FILE_INVALID) - Error(eDLL_T::RTECH, EXIT_FAILURE, "Couldn't open file \"%s\".\n", pakPatchPath); - - if (numBytesToProcess < pak->memoryData.patchHeaders[patchCount].compressedSize) - Error(eDLL_T::RTECH, EXIT_FAILURE, "File \"%s\" appears truncated; read size: %zu < expected size: %zu.\n", - pakPatchPath, numBytesToProcess, pak->memoryData.patchHeaders[patchCount].compressedSize); - - FS_CloseAsyncFile((short)fileStream->fileHandle); - - v43 = fileStream->dwordB8; - fileStream->fileHandle = v56; - v58 = (unsigned __int64)((v43 + 7) & 0xFFFFFFF8) << 19; - fileStream->qword0 = v58; - fileStream->byteBC = (v43 == ((v43 + 7) & 0xFFFFFFF8)) + 1; - fileStream->qword8 = v58 + pak->memoryData.patchHeaders[patchCount].compressedSize; - LABEL_84: - if (v43 == v44) - return memoryData->patchSrcSize == 0; } - - fileStream->gap14[v47] = -2; - fileStream->gap94[v47] = 1; - - if ((((_BYTE)v47 + 1) & 7) == 0) - fileStream->byteBC = 2; - - LABEL_65: - v43 = ++fileStream->dwordB8; - v62 = v48; - goto LABEL_84; } } @@ -605,7 +525,7 @@ LABEL_45: // if this is a header page, fetch info from the next unprocessed asset and copy over the asset's header bool SetupNextPageForPatching(PakLoadedInfo_t* a1, PakFile_t* pak) { - Rebuild_14043E030(pak); + Pak_RunAssetLoadingJobs(pak); // numProcessedPointers has just been set in the above function call pak->memoryData.numShiftedPointers = pak->numProcessedPointers; @@ -638,7 +558,7 @@ bool SetupNextPageForPatching(PakLoadedInfo_t* a1, PakFile_t* pak) int assetTypeIdx = pakAsset->HashTableIndexForAssetType(); pak->memoryData.patchDstPtr = reinterpret_cast(a1->segmentBuffers[0]) + pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx]; - pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx] += g_pPakGlobals->assetBindings[assetTypeIdx].nativeClassSize; + pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx] += g_pakGlobals->assetBindings[assetTypeIdx].nativeClassSize; return true; } @@ -741,7 +661,7 @@ bool Pak_ProcessAssets(PakLoadedInfo_t* const a1) pak->memoryData.patchDstPtr = reinterpret_cast(a1->segmentBuffers[0]) + pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx]; - pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx] += g_pPakGlobals->assetBindings[assetTypeIdx].nativeClassSize; + pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx] += g_pakGlobals->assetBindings[assetTypeIdx].nativeClassSize; } else { @@ -754,7 +674,7 @@ bool Pak_ProcessAssets(PakLoadedInfo_t* const a1) } } - if (!JT_IsJobDone(pak->memoryData.unkJobID)) + if (!JT_IsJobDone(pak->memoryData.assetLoadJobId)) return false; uint32_t i = 0; @@ -767,44 +687,55 @@ bool Pak_ProcessAssets(PakLoadedInfo_t* const a1) { //printf("[%s] processing deps for %llX (%.4s)\n", pak->GetName(), pAsset->guid, (char*)&pAsset->magic); Pak_ResolveAssetRelations(pak, pAsset); - const int v36 = pak->memoryData.qword2E0[i]; - if (dword_167A40B3C[6 * g_pPakGlobals->assets[v36].unk_8] == j) + const int assetIndex = pak->memoryData.loadedAssetIndices[i]; + const PakAssetShort_t& loadedAsset = g_pakGlobals->loadedAssets[assetIndex]; + + if (g_pakGlobals->trackedAssets[loadedAsset.trackerIndex].loadedPakIndex == j) { - if (*qword_167ED7BC8) + PakTracker_s* pakTracker = g_pakGlobals->pakTracker; + + if (pakTracker) { - uint64_t v38 = 0; - if ((*qword_167ED7BC8)->unk_0) + if (pakTracker->numPaksTracked) { - int* v39 = (*qword_167ED7BC8)->unk_array_9D410; - while (*v39 != v36) + int* trackerIndices = g_pakGlobals->pakTracker->loadedAssetIndices; + uint32_t count = 0; + + while (*trackerIndices != assetIndex) { - ++v38; - ++v39; - if (v38 >= (*qword_167ED7BC8)->unk_0) + ++count; + ++trackerIndices; + + if (count >= pakTracker->numPaksTracked) goto LABEL_41; } + goto LABEL_42; } } else { - //printf("allocating thing\n"); - *qword_167ED7BC8 = reinterpret_cast(AlignedMemAlloc()->Alloc(0x11D410, 8)); - (*qword_167ED7BC8)->unk_0 = 0; - (*qword_167ED7BC8)->unk_4 = 0; - (*qword_167ED7BC8)->unk_8 = 0; + pakTracker = reinterpret_cast(AlignedMemAlloc()->Alloc(sizeof(PakTracker_s), 8)); + pakTracker->numPaksTracked = 0; + pakTracker->unk_4 = 0; + pakTracker->unk_8 = 0; + + g_pakGlobals->pakTracker = pakTracker; } LABEL_41: - (*qword_167ED7BC8)->unk_array_9D410[(*qword_167ED7BC8)->unk_0] = v36; - ++(*qword_167ED7BC8)->unk_0; + + pakTracker->loadedAssetIndices[pakTracker->numPaksTracked] = assetIndex; + ++pakTracker->numPaksTracked; } } LABEL_42: ++i; } - if (*qword_167ED7BC8) + + if (g_pakGlobals->pakTracker) sub_14043D870(a1, 0); + a1->status = EPakStatus::PAK_STATUS_LOADED; return true; @@ -820,7 +751,7 @@ void Pak_StubInvalidAssetBinds(PakFile_t* const pak, PakSegmentDescriptor_t* con const uint8_t assetTypeIndex = asset->HashTableIndexForAssetType(); desc->assetTypeCount[assetTypeIndex]++; - PakAssetBinding_t* const assetBinding = &g_pPakGlobals->assetBindings[assetTypeIndex]; + PakAssetBinding_t* const assetBinding = &g_pakGlobals->assetBindings[assetTypeIndex]; if (assetBinding->type == PakAssetBinding_t::NONE) { @@ -897,10 +828,8 @@ bool Pak_StartLoadingPak(PakLoadedInfo_t* const loadedInfo) const PakFileHeader_t& pakHdr = pakFile->GetHeader(); - if (*g_pUseAssetStreamingSystem) - { + if (Pak_StreamingEnabled()) Pak_LoadStreamingData(loadedInfo); - } const __int64 v106 = pakHdr.descriptorCount + 2 * (pakHdr.patchIndex + pakHdr.assetCount + 4ull * pakHdr.assetCount + pakHdr.virtualSegmentCount); const __int64 patchDestOffset = pakHdr.GetTotalHeaderSize() + 2 * (pakHdr.patchIndex + 6ull * pakHdr.memPageCount + 4 * v106); @@ -917,6 +846,456 @@ bool Pak_StartLoadingPak(PakLoadedInfo_t* const loadedInfo) return true; } +// #STR: "(%02u).rpak", "Couldn't read package file \"%s\".\n", "Error 0x%08x", "paks\\Win64\\%s" +//bool Pak_SetupBuffersAndLoad(PakHandle_t pakId) +//{ +// __int64 v2; // rdi +// const char* pakFilePath; // rsi +// unsigned int v7; // ebx +// unsigned int v8; // r14d +// char** v9; // r12 +// int v10; // eax +// __int64 v11; // r11 +// char v12; // al +// char* i; // rdx +// uint32_t pakFileHandle; // eax +// bool result; // al +// int AsyncFile; // bl +// signed __int64 v19; // rbx +// char v20; // r14 +// __int64 v21; // rdx +// const char* v22; // rax +// __int64 v23; // rbx +// __int64 asset_entry_count_var; // rax MAPDST +// __int64 patchIndex; // r11 +// __int64 memPageCount; // r9 +// __int64 virtualSegmentCount; // rcx +// __int64 v32; // r8 +// __int64 v33; // rcx +// __int64 v34; // r14 +// __int64 v35; // r12 +// uint64_t ringBufferStreamSize; // rsi +// uint64_t ringBufferOutSize; // rax +// PakFile_t* pak; // rax MAPDST +// _DWORD* v40; // rax +// __int64 v43; // rdx +// uint8_t** v44; // rcx +// PakAsset_t** v45; // rdx +// PakPatchFileHeader_t* p_patchHeader; // rcx +// uint16_t v47; // ax +// PakPatchFileHeader_t* p_decompressedSize; // rdx +// __int64 v49; // rcx +// __int64 v50; // rax +// unsigned __int16* v51; // rcx +// char* v52; // rcx +// char* v53; // rdx +// __int64 v54; // rax +// PakSegmentHeader_t* v55; // rdx +// __int64 v56; // rcx +// __int64 v57; // rax +// PakPageHeader_t* v58; // rcx +// __int64 descriptorCount; // rdx +// PakPage_t* v60; // rcx +// __int64 assetCount; // rax +// PakAsset_t* v62; // r8 +// __int64 guidDescriptorCount; // r9 +// PakPage_t* v64; // rdx +// __int64 relationsCounts; // rcx +// uint32_t* v66; // rax +// __int64 v67; // rdx +// uint32_t* v68; // rcx +// __int64 v69; // r8 +// uint32_t* v70; // rax +// __int64 v71; // rcx +// PakPatchDataHeader_t* patchDataHeader; // rax +// size_t v73; // rdx +// uint8_t* ringBuffer; // rax +// bool v75; // zf +// __int64 v76; // r10 +// unsigned __int64 v77; // r8 +// __int64 v78; // rax +// PakFileHeader_t pakHdr; // [rsp+40h] [rbp-C0h] BYREF +// __int64 v80; // [rsp+C0h] [rbp-40h] +// __int64 v81; // [rsp+C8h] [rbp-38h] +// char relativeFilePath[260]; // [rsp+E0h] [rbp-20h] BYREF +// char v84[12]; // [rsp+1E4h] [rbp+E4h] BYREF +// CHAR LibFileName[4]; // [rsp+1F0h] [rbp+F0h] BYREF +// char v86[8252]; // [rsp+1F4h] [rbp+F4h] +// size_t pak_file_size_var; // [rsp+2248h] [rbp+2148h] BYREF +// __int64 v89; // [rsp+2250h] [rbp+2150h] +// uint64_t v90; // [rsp+2258h] [rbp+2158h] +// +// v2 = 0i64; +// +// PakLoadedInfo_t* const loadedInfo = Pak_GetPakInfo(pakId); +// +// loadedInfo->status = PAK_STATUS_LOAD_STARTING; +// pakFilePath = loadedInfo->fileName; +// +// // +// assert(pakFilePath); +// +// const char* nameUnqualified = V_UnqualifiedFileName(pakFilePath); +// +// if (nameUnqualified != pakFilePath) +// goto LABEL_27; +// +// snprintf(relativeFilePath, 0x100ui64, "paks\\Win64\\%s", pakFilePath); +// +// v7 = g_nPatchEntryCount; +// v8 = 0; +// +// // if this pak is patched, load the last patch file first before proceeding +// // with any other pak that is getting patched. note that the patch number +// // does not indicate which pak file is the actual last patch file; a patch +// // with number (01) can also patch (02) and base, these details are +// // determined from the pak file header +// if (g_nPatchEntryCount) +// { +// v9 = g_pszPakPatchList; +// +// int n; +// while (1) +// { +// n = V_stricmp(pakFilePath, v9[(v7 + v8) >> 1]); +// +// if (n >= 0) +// break; +// +// v7 = v11; +// LABEL_13: +// if (v8 >= v7) +// goto SKIP_PATCH_LOADING; +// } +// +// if (n > 0) +// { +// v8 = v11 + 1; +// goto LABEL_13; +// } +// +// const int patchNumber = g_pnPatchNumbers[v11]; +// +// if (patchNumber) +// { +// char* const extension = (char*)V_GetFileExtension(pakFilePath, true); +// snprintf(extension, &relativeFilePath[sizeof(relativeFilePath)] - extension, "(%02u).rpak", patchNumber); +// } +// } +// +// if (!g_nPatchEntryCount) +// goto SKIP_PATCH_LOADING; +// +// v9 = g_pszPakPatchList; +// while (1) +// { +// LOBYTE(v10) = stricmp(pakFilePath, v9[(v7 + v8) >> 1]); +// if (v10 >= 0) +// break; +// v7 = v11; +// LABEL_13: +// if (v8 >= v7) +// goto SKIP_PATCH_LOADING; +// } +// if (v10 > 0) +// { +// v8 = v11 + 1; +// goto LABEL_13; +// } +// +// const int patchNumber = g_pnPatchNumbers[v11]; +// +// if (patchNumber) +// { +// char* const extension = (char*)V_GetFileExtension(pakFilePath, true); +// snprintf(extension, &relativeFilePath[sizeof(relativeFilePath)] - extension, "(%02u).rpak", patchNumber); +// } +// +//SKIP_PATCH_LOADING: +// pakFilePath = relativeFilePath; +//LABEL_27: +// pakFileHandle = FS_OpenAsyncFile(pakFilePath, loadedInfo->logLevel, &pak_file_size_var); +// +// if (pakFileHandle == FS_ASYNC_FILE_INVALID) +// { +// if (async_debug_level->GetInt() >= loadedInfo->logLevel) +// Error(eDLL_T::RTECH, NO_ERROR, "Couldn't read package file \"%s\".\n", pakFilePath); +// +// loadedInfo->status = PAK_STATUS_ERROR; +// return false; +// } +// +// loadedInfo->fileHandle = pakFileHandle; +// +// // file appears truncated/corrupt +// if (pak_file_size_var < sizeof(PakFileHeader_t)) +// { +// loadedInfo->status = PAK_STATUS_ERROR; +// return false; +// } +// +// AsyncFile = v_FS_ReadAsyncFile(pakFileHandle, 0i64, sizeof(PakFileHeader_t), &pakHdr, 0i64, 0i64, 4); +// v_FS_WaitForAsyncRead(AsyncFile); +// +// size_t bytesProcessed = 0; +// const char* statusMsg = "(no reason)"; +// const uint8_t currentStatus = g_pakLoadApi->CheckAsyncRequest(AsyncFile, &bytesProcessed, &statusMsg); +// +// if (currentStatus == AsyncHandleStatus_t::FS_ASYNC_ERROR) +// { +// Error(eDLL_T::RTECH, EXIT_FAILURE, "Error reading pak file \"%s\" -- %s\n", pak->memoryData.fileName, statusMsg); +// +// loadedInfo->status = PAK_STATUS_ERROR; +// return false; +// } +// +// if (pakHdr.magic != PAK_HEADER_MAGIC || pakHdr.version != PAK_HEADER_VERSION) +// { +// loadedInfo->status = PAK_STATUS_ERROR; +// return false; +// } +// +// if (pakHdr.flags & PAK_HEADER_FLAGS_HAS_MODULE_EXTENDED) +// { +// v22 = V_GetFileExtension(pakFilePath); +// v23 = v22 - pakFilePath; +// +// if ((unsigned __int64)(v22 - pakFilePath + 4) >= 0x2000) +// { +// loadedInfo->status = PAK_STATUS_ERROR; +// return false; +// } +// +// memcpy(LibFileName, pakFilePath, v22 - pakFilePath); +// +// *(_DWORD*)&LibFileName[v23] = *(_DWORD*)".dll"; +// v86[v23] = '\0'; +// +// const HMODULE hModule = LoadLibraryA(LibFileName); +// loadedInfo->hModule = hModule; +// +// if (!hModule) +// { +// loadedInfo->status = PAK_STATUS_ERROR; +// return false; +// } +// } +// +// loadedInfo->fileTime = pakHdr.fileTime; +// asset_entry_count_var = pakHdr.assetCount; +// +// loadedInfo->assetCount = pakHdr.assetCount; +// +// asset_entry_count_var = pakHdr.assetCount; +// patchIndex = pakHdr.patchIndex; +// memPageCount = pakHdr.memPageCount; +// virtualSegmentCount = pakHdr.virtualSegmentCount; +// v32 = *(unsigned int*)&pakHdr.unk2[4]; +// +// loadedInfo->assetGuids = (PakGuid_t*)loadedInfo->allocator->Alloc(sizeof(PakGuid_t) * asset_entry_count_var, 8);; +// +// size_t streamingFilesBuifSize = pakHdr.streamingFilesBufSize[STREAMING_SET_OPTIONAL] + pakHdr.streamingFilesBufSize[STREAMING_SET_MANDATORY]; +// +// v81 = 8 * memPageCount; +// +// pak_file_size_var = streamingFilesBuifSize +// + ((_WORD)patchIndex != 0 ? 8 : 0) +// + 2 +// * (patchIndex +// + 2 +// * (pakHdr.relationsCounts +// + *(unsigned int*)pakHdr.unk2 +// + 3 * memPageCount +// + 2 * (pakHdr.descriptorCount + pakHdr.guidDescriptorCount + 16i64 + 2 * (asset_entry_count_var + patchIndex + 4 * asset_entry_count_var + virtualSegmentCount)))) +// + v32; +// +// v80 = 4 * asset_entry_count_var; +// v90 = pak_file_size_var + 2080; +// v33 = -((_DWORD)pak_file_size_var + 2080 + 4 * (_DWORD)asset_entry_count_var) & 7; +// v89 = v33; +// v34 = 4 * asset_entry_count_var + pak_file_size_var + 2080 + v33 + 8 * memPageCount + 12 * asset_entry_count_var; +// v35 = (-(4 * (_DWORD)asset_entry_count_var + (_DWORD)pak_file_size_var + 2080 + (_DWORD)v33 + 8 * (_DWORD)memPageCount + 12 * (_DWORD)asset_entry_count_var) & 7) + 4088i64; +// +// if ((pakHdr.flags & 0x100) != 0) +// { +// ringBufferStreamSize = PAK_DECODE_IN_RING_BUFFER_SIZE; +// ringBufferOutSize = PAK_DECODE_OUT_RING_BUFFER_SIZE; +// +// if (pakHdr.compressedSize < PAK_DECODE_IN_RING_BUFFER_SIZE && !(_WORD)patchIndex) +// ringBufferStreamSize = (pakHdr.compressedSize + PAK_DECODE_IN_RING_BUFFER_SMALL_MASK) & 0xFFFFFFFFFFFFF000ui64; +// } +// else +// { +// ringBufferStreamSize = 0i64; +// ringBufferOutSize = PAK_DECODE_IN_RING_BUFFER_SIZE; +// } +// +// if (ringBufferOutSize > pakHdr.decompressedSize && !(_WORD)patchIndex) +// ringBufferOutSize = (pakHdr.decompressedSize + PAK_DECODE_IN_RING_BUFFER_SMALL_MASK) & 0xFFFFFFFFFFFFF000ui64; +// +// pak = (PakFile_t*)AlignedMemAlloc()->Alloc(v34 + v35 + ringBufferOutSize + ringBufferStreamSize, 8); +// +// if (pak) +// { +// loadedInfo->pakFile = pak; +// +// *(_QWORD*)&pak->processedPageCount = 0i64; +// *(_QWORD*)&pak->patchCount = 0i64; +// *(_QWORD*)&pak->numProcessedPointers = 0i64; +// *(_QWORD*)&pak->memoryData.someAssetCount = 0i64; +// +// v40 = (_DWORD*)loadedInfo->allocator->Alloc(16i64 * pakHdr.guidDescriptorCount + 8, 8i64); +// loadedInfo->qword50 = v40; +// +// *v40 = 0; +// +// *(_DWORD*)(loadedInfo->qword50 + 4i64) = 0; +// +// pak->memoryData.pakHeader = pakHdr; +// +// pak->memoryData.pakId = pakId; +// pak->memoryData.qword2D0 = pak_file_size_var; +// +// JobID_t jobId = PAK_DEFAULT_JOB_GROUP_ID; +// +// if (pakHdr.assetCount) +// jobId = JT_BeginJobGroup(0); +// +// pak->memoryData.unkJobID = jobId; +// +// +// v43 = v81; +// v44 = (uint8_t**)((char*)pak + v89 + v90 + v80); +// pak->memoryData.memPageBuffers = v44; +// v45 = (PakAsset_t**)((char*)v44 + v43); +// pak->memoryData.ppAssetEntries = v45; +// pak->memoryData.qword2E0 = (int*)&v45[pakHdr.assetCount]; +// p_patchHeader = &pak->memoryData.patchHeader; +// v47 = pak->memoryData.pakHeader.patchIndex; +// +// p_decompressedSize = (PakPatchFileHeader_t*)&pak->memoryData.patchHeader.decompressedSize; +// if (!v47) +// p_decompressedSize = &pak->memoryData.patchHeader; +// if (!v47) +// p_patchHeader = 0i64; +// +// pak->memoryData.patchDataHeader = (PakPatchDataHeader_t*)p_patchHeader; +// v49 = pak->memoryData.pakHeader.patchIndex; +// pak->memoryData.patchHeaders = p_decompressedSize; +// v50 = pak->memoryData.pakHeader.patchIndex; +// v51 = (unsigned __int16*)&p_decompressedSize[v49]; +// pak->memoryData.patchIndices = v51; +// v52 = (char*)&v51[v50]; +// v53 = &v52[pak->memoryData.pakHeader.streamingFilesBufSize[0]]; +// pak->memoryData.streamingFilePaths[0] = v52; +// v54 = pak->memoryData.pakHeader.streamingFilesBufSize[1]; +// pak->memoryData.streamingFilePaths[1] = v53; +// v55 = (PakSegmentHeader_t*)&v53[v54]; +// v56 = pak->memoryData.pakHeader.virtualSegmentCount; +// pak->memoryData.segmentHeaders = v55; +// v57 = pak->memoryData.pakHeader.memPageCount; +// v58 = (PakPageHeader_t*)&v55[v56]; +// pak->memoryData.pageHeaders = v58; +// descriptorCount = pak->memoryData.pakHeader.descriptorCount; +// v60 = (PakPage_t*)&v58[v57]; +// pak->memoryData.virtualPointers = v60; +// assetCount = pak->memoryData.pakHeader.assetCount; +// v62 = (PakAsset_t*)&v60[descriptorCount]; +// pak->memoryData.assetEntries = v62; +// guidDescriptorCount = pak->memoryData.pakHeader.guidDescriptorCount; +// v64 = (PakPage_t*)&v62[assetCount]; +// pak->memoryData.guidDescriptors = v64; +// relationsCounts = pak->memoryData.pakHeader.relationsCounts; +// v66 = (uint32_t*)&v64[guidDescriptorCount]; +// pak->memoryData.fileRelations = v66; +// v67 = *(unsigned int*)pak->memoryData.pakHeader.unk2; +// v68 = &v66[relationsCounts]; +// *(_QWORD*)pak->memoryData.gap5E0 = v68; +// v69 = *(unsigned int*)&pak->memoryData.pakHeader.unk2[4]; +// *(_QWORD*)&pak->memoryData.gap5E0[16] = 0i64; +// v70 = &v68[v67]; +// *(_QWORD*)&pak->memoryData.gap5E0[8] = v70; +// v71 = (__int64)v70 + v69; +// *(_QWORD*)&pak->memoryData.gap5E0[24] = (char*)v70 + v69; +// if (pak->memoryData.pakHeader.patchIndex) +// { +// patchDataHeader = pak->memoryData.patchDataHeader; +// if (patchDataHeader->editStreamSize) +// { +// *(_QWORD*)&pak->memoryData.gap5E0[16] = v71; +// *(_QWORD*)&pak->memoryData.gap5E0[24] = v71 + (unsigned int)patchDataHeader->editStreamSize; +// } +// } +// v73 = PAK_DECODE_IN_RING_BUFFER_MASK; +// +// pak->memoryData.fileName = loadedInfo->fileName; +// pak->fileStream.qword0 = 0i64; +// pak->fileStream.fileSize = pakHdr.compressedSize; +// pak->fileStream.fileHandle = loadedInfo->fileHandle; +// loadedInfo->fileHandle = FS_ASYNC_FILE_INVALID; +// +// pak->fileStream.bufferMask = PAK_DECODE_IN_RING_BUFFER_MASK; +// +// ringBuffer = (uint8_t*)(((unsigned __int64)pak + v35 + v34) & 0xFFFFFFFFFFFFF000ui64); +// +// pak->fileStream.buffer = ringBuffer; +// pak->fileStream.numDataChunksProcessed = 0; +// pak->fileStream.numDataChunks = 0; +// pak->fileStream.fileReadStatus = 3; +// pak->fileStream.finishedLoadingPatches = 0; +// pak->fileStream.numLoadedFiles = 0; +// pak->fileStream.bytesStreamed = sizeof(PakFileHeader_t); +// pak->decompBuffer = &ringBuffer[ringBufferStreamSize]; +// pak->inputBytePos = sizeof(PakFileHeader_t); +// pak->byte1F8 = 0; +// pak->byte1FD = 1; +// pak->isOffsetted_MAYBE = 0; +// pak->isCompressed = 0; +// +// // FINISHME: this means if the pak file is not encoded, but we should also +// // check on the zstd flags +// v75 = (pakHdr.flags & 0x100) == 0; +// +// pak->qword298 = 128i64; +// +// if (!v75) +// v73 = PAK_DECODE_OUT_RING_BUFFER_MASK; +// +// pak->maxCopySize = v73; +// memset(&pak->pakDecoder, 0, sizeof(pak->pakDecoder)); +// +// pak->pakDecoder.outBufBytePos = 128i64; +// pak->pakDecoder.decompSize = 128i64; +// pak->memoryData.processedPatchedDataSize = 128i64; +// v76 = pakHdr.patchIndex; +// v77 = pakHdr.descriptorCount + 2 * (pakHdr.patchIndex + (unsigned __int64)pakHdr.assetCount + 4i64 * pakHdr.assetCount + pakHdr.virtualSegmentCount); +// v78 = pakHdr.memPageCount; +// +// pak->memoryData.field_2A8 = 0i64; +// pak->memoryData.patchData = 0i64; +// pak->memoryData.patchDataPtr = 0i64; +// pak->memoryData.bitBuf.m_dataBuf = 0i64; +// pak->memoryData.bitBuf.m_bitsAvailable = 0; +// pak->memoryData.patchDataOffset = 0; +// pak->memoryData.patchSrcSize = v82 + ((_WORD)v76 != 0 ? 8 : 0) + 2 * (v76 + 6 * v78 + 4 * v77); +// pak->memoryData.patchDstPtr = (char*)&pak->memoryData.patchHeader; +// pak->memoryData.patchFunc = g_pakPatchApi[0]; +// +// LOBYTE(v2) = pakHdr.patchIndex == 0; +// pak->memoryData.numBytesToProcess_maybe = pak->memoryData.pakHeader.decompressedSize + v2 - 0x80; +// +// Pak_ProcessPakFile(pak); +// return true; +// } +// else +// { +// loadedInfo->status = PAK_STATUS_ERROR; +// return false; +// } +// +// return result; +//} + void V_PakParse::Detour(const bool bAttach) const { @@ -929,7 +1308,7 @@ void V_PakParse::Detour(const bool bAttach) const DetourSetup(&v_Pak_ResolveAssetRelations, &Pak_ResolveAssetRelations, bAttach); DetourSetup(&v_Pak_ProcessAssets, &Pak_ProcessAssets, bAttach); - DetourSetup(&sub_14043E030, &Rebuild_14043E030, bAttach); + DetourSetup(&v_Pak_RunAssetLoadingJobs, &Pak_RunAssetLoadingJobs, bAttach); } // Symbols taken from R2 dll's. diff --git a/r5dev/rtech/pak/pakparse.h b/r5dev/rtech/pak/pakparse.h index 84df8ed9..22d62df3 100644 --- a/r5dev/rtech/pak/pakparse.h +++ b/r5dev/rtech/pak/pakparse.h @@ -20,19 +20,14 @@ inline bool(*v_Pak_ProcessPakFile)(PakFile_t* const pak); inline bool(*v_Pak_ProcessAssets)(PakLoadedInfo_t* pakInfo); inline void(*v_Pak_ResolveAssetRelations)(PakFile_t* const pak, const PakAsset_t* const asset); +inline void (*v_Pak_RunAssetLoadingJobs)(PakFile_t* pak); +inline void (*Pak_ProcessAssetRelationsAndResolveDependencies)(PakFile_t* pak_arg, PakAsset_t* asset_arg, unsigned int asset_idx_arg, unsigned int a4); + +inline int (*Pak_TrackAsset)(PakFile_t* const a1, PakAsset_t* a2); + // TODO: name these! -inline void (*sub_14043E030)(PakFile_t* pak); -inline __int64 (*sub_14043D3C0)(PakFile_t* a1, PakAsset_t* a2); -inline void (*sub_14043D150)(PakFile_t* pak_arg, PakAsset_t* asset_arg, unsigned int asset_idx_arg, unsigned int a4); -inline void (*sub_14045B310)(unsigned int a1, __int64 a2); inline void (*sub_14043D870)(PakLoadedInfo_t* a1, int a2); -inline short* word_167ED7BDE = nullptr; - -// potentially PakAssetShort_t -inline int* dword_167A40B3C = nullptr; -inline UnknownPakStruct_t** qword_167ED7BC8 = nullptr; // ptr to buffer with size 0x11D410 - typedef struct PakLoadFuncs_s { void* Initialize; // Returns the pak handle of the patch master RPak once initialized. @@ -66,8 +61,8 @@ typedef struct PakLoadFuncs_s void* Func25; void* ReadAsyncFile; void* ReadAsyncFileWithUserData; - uint8_t (*CheckAsyncRequest)(unsigned char idx, size_t* const bytesProcessed, const char** const statusMsg); - uint8_t (*WaitAndCheckAsyncRequest)(unsigned char idx, size_t* const bytesProcessed, const char** const statusMsg); + uint8_t (*CheckAsyncRequest)(int idx, size_t* const bytesProcessed, const char** const statusMsg); + uint8_t (*WaitAndCheckAsyncRequest)(int idx, size_t* const bytesProcessed, const char** const statusMsg); void* WaitForAsyncFileRead; void* Func31; void* Func32; @@ -95,7 +90,6 @@ class V_PakParse : public IDetour LogFunAdr("Pak_ProcessAssets", v_Pak_ProcessAssets); LogFunAdr("Pak_ResolveAssetRelations", v_Pak_ResolveAssetRelations); - LogVarAdr("word_167ED7BDE", word_167ED7BDE); LogVarAdr("g_pakLoadApi", g_pakLoadApi); } virtual void GetFun(void) const @@ -111,22 +105,15 @@ class V_PakParse : public IDetour g_GameDll.FindPatternSIMD("E8 ?? ?? ?? ?? 84 C0 0F 84 ?? ?? ?? ?? 48 8B CF E8 ?? ?? ?? ?? 44 0F B7 05 ?? ?? ?? ??").FollowNearCallSelf().GetPtr(v_Pak_ProcessAssets); g_GameDll.FindPatternSIMD("E8 ?? ?? ?? ?? 48 8B 86 ?? ?? ?? ?? 42 8B 0C B0").FollowNearCallSelf().GetPtr(v_Pak_ResolveAssetRelations); - g_GameDll.FindPatternSIMD("40 53 56 48 83 EC 58 44 8B 09").GetPtr(sub_14043E030); - g_GameDll.FindPatternSIMD("48 89 5C 24 ?? 48 89 6C 24 ?? 48 89 74 24 ?? 57 41 54 41 55 41 56 41 57 48 83 EC 20 44 8B 0D ?? ?? ?? ?? 4C 8B E9").GetPtr(sub_14043D3C0); - g_GameDll.FindPatternSIMD("48 89 5C 24 ?? 48 89 6C 24 ?? 48 89 74 24 ?? 57 48 83 EC 30 41 8B E9").GetPtr(sub_14043D150); - g_GameDll.FindPatternSIMD("48 89 5C 24 ?? 57 48 83 EC 20 83 7A 14 01").GetPtr(sub_14045B310); + g_GameDll.FindPatternSIMD("40 53 56 48 83 EC 58 44 8B 09").GetPtr(v_Pak_RunAssetLoadingJobs); + g_GameDll.FindPatternSIMD("48 89 5C 24 ?? 48 89 6C 24 ?? 48 89 74 24 ?? 57 41 54 41 55 41 56 41 57 48 83 EC 20 44 8B 0D ?? ?? ?? ?? 4C 8B E9").GetPtr(Pak_TrackAsset); + g_GameDll.FindPatternSIMD("48 89 5C 24 ?? 48 89 6C 24 ?? 48 89 74 24 ?? 57 48 83 EC 30 41 8B E9").GetPtr(Pak_ProcessAssetRelationsAndResolveDependencies); g_GameDll.FindPatternSIMD("E8 ?? ?? ?? ?? EB 14 48 8D 0D ?? ?? ?? ??").FollowNearCallSelf().GetPtr(sub_14043D870); } virtual void GetVar(void) const { g_pakLoadApi = CMemory(v_LauncherMain).Offset(0x820).FindPatternSelf("48 89").ResolveRelativeAddressSelf(0x3, 0x7).RCast(); - - CMemory(sub_14043E030).FindPatternSelf("66 44 39").ResolveRelativeAddressSelf(0x4, 0x8).GetPtr(word_167ED7BDE); - int* va_dword_167A40B3C = CMemory(v_Pak_ProcessAssets).Offset(0x200).FindPatternSelf("44 39 BC").Offset(4).RCast(); - dword_167A40B3C = reinterpret_cast(g_GameDll.GetModuleBase() + *va_dword_167A40B3C); - - CMemory(v_Pak_ProcessAssets).Offset(0x200).FindPatternSelf("48 8B 15").ResolveRelativeAddressSelf(0x3, 0x7).GetPtr(qword_167ED7BC8); } virtual void GetCon(void) const { } virtual void Detour(const bool bAttach) const; diff --git a/r5dev/rtech/pak/pakstate.h b/r5dev/rtech/pak/pakstate.h index 45f34ae0..f3efb868 100644 --- a/r5dev/rtech/pak/pakstate.h +++ b/r5dev/rtech/pak/pakstate.h @@ -6,63 +6,34 @@ #include "rtech/ipakfile.h" -inline PakGlobals_t* g_pPakGlobals; -inline PakLoadedInfo_t* g_pLoadedPakInfo; +inline PakGlobals_s* g_pakGlobals; +inline JobHelpCallback_t g_pPakFifoLockWrapper; // Pointer to functor that takes the global pak fifolock as argument. -inline JobID_t* g_pPakLoadJobID; - -inline int16_t* g_pLoadedPakCount; -inline int16_t* g_pRequestedPakCount; - -inline JobFifoLock_s* g_pPakFifoLock; -inline void* g_pPakFifoLockWrapper; // Pointer to functor that takes the global pak fifolock as argument. +// TODO: rename to 'g_bPakFifoLockAcquiredInMainThread' +// if this is set, JT_ReleaseFifoLock has to be called +// twice as the depth goes up to the thread that +// acquired the lock + the main thread inline bool* g_bPakFifoLockAcquired; -// bool as int64 -inline int64_t* g_pPakHasPendingUnloadJobs; - /////////////////////////////////////////////////////////////////////////////// class V_PakState : public IDetour { virtual void GetAdr(void) const { - LogVarAdr("g_pakGlobals", g_pPakGlobals); - LogVarAdr("g_loadedPakInfo", g_pLoadedPakInfo); + LogVarAdr("g_pakGlobals", g_pakGlobals); - LogVarAdr("g_pakLoadJobID", g_pPakLoadJobID); - - LogVarAdr("g_loadedPakCount", g_pLoadedPakCount); - LogVarAdr("g_requestedPakCount", g_pRequestedPakCount); - - LogVarAdr("g_pakFifoLock", g_pPakFifoLock); LogVarAdr("g_pakFifoLockWrapper", g_pPakFifoLockWrapper); LogVarAdr("g_pakFifoLockAcquired", g_bPakFifoLockAcquired); - - LogVarAdr("g_pakHasPendingUnloadJobs", g_pPakHasPendingUnloadJobs); } virtual void GetFun(void) const { } virtual void GetVar(void) const { - extern void(*v_Pak_UnloadAsync)(PakHandle_t); - const CMemory pakUnloadBase(v_Pak_UnloadAsync); - - g_pLoadedPakInfo = pakUnloadBase.FindPattern("48 8D 05", CMemory::Direction::DOWN).ResolveRelativeAddressSelf(0x3, 0x7).RCast(); - g_pRequestedPakCount = pakUnloadBase.FindPattern("66 89", CMemory::Direction::DOWN, 450).ResolveRelativeAddressSelf(0x3, 0x7).RCast(); - g_pLoadedPakCount = &*g_pRequestedPakCount - 1; // '-1' shifts it back with sizeof(int16_t). - - g_pPakGlobals = g_GameDll.FindPatternSIMD("48 8D 1D ?? ?? ?? ?? 45 8D 5A 0E").ResolveRelativeAddressSelf(0x3, 0x7).RCast(); /*48 8D 1D ? ? ? ? 45 8D 5A 0E*/ - g_pPakLoadJobID = reinterpret_cast(&*g_pLoadedPakCount - 2); + g_pakGlobals = g_GameDll.FindPatternSIMD("48 8D 1D ?? ?? ?? ?? 45 8D 5A 0E").ResolveRelativeAddressSelf(0x3, 0x7).RCast(); /*48 8D 1D ? ? ? ? 45 8D 5A 0E*/ const CMemory jtBase(JT_HelpWithAnything); - g_pPakFifoLock = jtBase.Offset(0x155).FindPatternSelf("48 8D 0D").ResolveRelativeAddressSelf(0x3, 0x7).RCast(); - g_pPakFifoLockWrapper = jtBase.Offset(0x1BC).FindPatternSelf("48 8D 0D").ResolveRelativeAddressSelf(0x3, 0x7).RCast(); + g_pPakFifoLockWrapper = jtBase.Offset(0x1BC).FindPatternSelf("48 8D 0D").ResolveRelativeAddressSelf(0x3, 0x7).RCast(); g_bPakFifoLockAcquired = jtBase.Offset(0x50).FindPatternSelf("C6 05").ResolveRelativeAddressSelf(0x2, 0x7).RCast(); - - extern EPakStatus(*v_Pak_WaitAsync)(PakHandle_t, void*); - const CMemory pakWaitBase(v_Pak_WaitAsync); - - pakWaitBase.Offset(0x80).FindPatternSelf("48 39").ResolveRelativeAddressSelf(3, 7).GetPtr(g_pPakHasPendingUnloadJobs); } virtual void GetCon(void) const { } virtual void Detour(const bool bAttach) const { }; diff --git a/r5dev/rtech/pak/pakstream.cpp b/r5dev/rtech/pak/pakstream.cpp index aa666756..f530c722 100644 --- a/r5dev/rtech/pak/pakstream.cpp +++ b/r5dev/rtech/pak/pakstream.cpp @@ -6,8 +6,10 @@ #include "tier0/commandline.h" #include "rtech/async/asyncio.h" #include "rtech/ipakfile.h" -#include "pakstream.h" + #include "pakparse.h" +#include "pakstate.h" +#include "pakstream.h" //----------------------------------------------------------------------------- // determines whether or not to emulate the streaming install, this basically @@ -16,23 +18,20 @@ //----------------------------------------------------------------------------- static bool Pak_ShouldEmulateStreamingInstall() { - static bool initialized = false; - static bool shouldEmulate = false; - // don't run the command line check every query - if (initialized) - return shouldEmulate; + if (g_pakGlobals->emulateStreamingInstallInit) + return g_pakGlobals->emulateStreamingInstall; const char* value = nullptr; if (CommandLine()->CheckParm("-emulate_streaming_install", &value)) { if (value && atoi(value)) - shouldEmulate = true; + g_pakGlobals->emulateStreamingInstall = true; } - initialized = true; - return shouldEmulate; + g_pakGlobals->emulateStreamingInstallInit = true; + return g_pakGlobals->emulateStreamingInstall; } //----------------------------------------------------------------------------- @@ -95,7 +94,7 @@ void Pak_EnableEmbeddedStreamingData(PakLoadedInfo_t* const loadedInfo, PakLoade const size_t basePathLen = hasPath ? 0 : strlen(PAK_BASE_PATH); const size_t totalBufLen = basePathLen + baseNameLen + 1; - char* const embeddedName = reinterpret_cast(loadedInfo->allocator->Alloc(totalBufLen, sizeof(char))); + char* const embeddedName = reinterpret_cast(loadedInfo->allocator->Alloc(totalBufLen, 1)); assert(embeddedName); // copy the base path if none was found in the file name diff --git a/r5dev/rtech/pak/pakstream.h b/r5dev/rtech/pak/pakstream.h index 6f43e215..7667acd2 100644 --- a/r5dev/rtech/pak/pakstream.h +++ b/r5dev/rtech/pak/pakstream.h @@ -1,6 +1,7 @@ #ifndef RTECH_PAKSTREAM_H #define RTECH_PAKSTREAM_H #include "rtech/ipakfile.h" +#include "pakstate.h" extern void Pak_OpenAssociatedStreamingFiles(PakLoadedInfo_t* const loadedInfo, PakLoadedInfo_t::StreamingInfo_t& streamInfo, const uint16_t fileNamesBufSize, const EPakStreamSet set); @@ -8,46 +9,31 @@ extern void Pak_OpenAssociatedStreamingFiles(PakLoadedInfo_t* const loadedInfo, extern void Pak_EnableEmbeddedStreamingData(PakLoadedInfo_t* const loadedInfo, PakLoadedInfo_t::StreamingInfo_t& streamInfo); extern void Pak_LoadStreamingData(PakLoadedInfo_t* const loadedInfo); -// bool set as int64. -inline int64_t* g_pUseAssetStreamingSystem = nullptr; -inline int64_t* g_pNumStreamableAssets = nullptr; +// the current download progress of optional streaming assets inline float* g_pStreamingDownloadProgress = nullptr; -// inlines -inline void(*v_Pak_IncrementStreamingAssetCount)(void); -inline void(*v_Pak_DecrementStreamingAssetCount)(void); +// NOTE: must use these when incrementing asset counts !!! +inline void Pak_IncrementStreamingAssetCount() { ThreadInterlockedIncrement64(&g_pakGlobals->numStreamableAssets); } +inline void Pak_DecrementStreamingAssetCount() { ThreadInterlockedDecrement64(&g_pakGlobals->numStreamableAssets); } -inline bool Pak_StreamingEnabled() { return *g_pUseAssetStreamingSystem != NULL; } -inline int64_t Pak_GetNumStreamableAssets() { return *g_pNumStreamableAssets; } +inline int64_t Pak_GetNumStreamableAssets() { return g_pakGlobals->numStreamableAssets; } inline float Pak_GetStreamingDownloadProgress() { return *g_pStreamingDownloadProgress; } inline bool Pak_StreamingDownloadFinished() { return Pak_GetStreamingDownloadProgress() == 1.0f; } +inline bool Pak_StreamingEnabled() { return g_pakGlobals->useStreamingSystem != NULL; } + class V_PakStream : public IDetour { virtual void GetAdr(void) const { - LogFunAdr("Pak_IncrementStreamingAssetCount", v_Pak_IncrementStreamingAssetCount); - LogFunAdr("Pak_DecrementStreamingAssetCount", v_Pak_DecrementStreamingAssetCount); - - LogVarAdr("g_useAssetStreamingSystem", g_pUseAssetStreamingSystem); - LogVarAdr("g_numStreamableAssets", g_pNumStreamableAssets); LogVarAdr("g_streamingDownloadProgress", g_pStreamingDownloadProgress); } - virtual void GetFun(void) const - { - g_GameDll.FindPatternSIMD("F0 48 FF 05 ?? ?? ?? ?? C3").GetPtr(v_Pak_IncrementStreamingAssetCount); - g_GameDll.FindPatternSIMD("F0 48 FF 0D ?? ?? ?? ?? C3").GetPtr(v_Pak_DecrementStreamingAssetCount); - } + virtual void GetFun(void) const { } virtual void GetVar(void) const { - extern PakHandle_t(*v_Pak_Initialize)(int mode); - CMemory(v_Pak_Initialize).Offset(0x120).FindPatternSelf("48 89 05").ResolveRelativeAddressSelf(3, 7).GetPtr(g_pUseAssetStreamingSystem); - g_GameDll.FindPatternSIMD("F3 0F 10 05 ?? ?? ?? ?? C3 CC CC CC CC CC CC CC 48 89 5C 24 ?? 57 48 81 EC ?? ?? ?? ??") .ResolveRelativeAddress(0x4, 0x8).GetPtr(g_pStreamingDownloadProgress); - - CMemory(v_Pak_IncrementStreamingAssetCount).ResolveRelativeAddress(4, 8).GetPtr(g_pNumStreamableAssets); // 167ED7BB8 } virtual void GetCon(void) const { } virtual void Detour(const bool bAttach) const { }; diff --git a/r5dev/rtech/pak/paktools.cpp b/r5dev/rtech/pak/paktools.cpp index 4a071825..a589e150 100644 --- a/r5dev/rtech/pak/paktools.cpp +++ b/r5dev/rtech/pak/paktools.cpp @@ -101,6 +101,20 @@ const char* Pak_StatusToString(const EPakStatus status) } } +//----------------------------------------------------------------------------- +// returns pak decoder as string +//----------------------------------------------------------------------------- +const char* Pak_DecoderToString(const EPakDecodeMode mode) +{ + switch (mode) + { + case EPakDecodeMode::MODE_RTECH: return "RTech"; + case EPakDecodeMode::MODE_ZSTD: return "ZStd"; + } + + UNREACHABLE(); +} + //----------------------------------------------------------------------------- // compute a guid from input string data //----------------------------------------------------------------------------- @@ -146,22 +160,9 @@ PakGuid_t Pak_StringToGuid(const char* const string) //----------------------------------------------------------------------------- // gets information about loaded pak file via pak id //----------------------------------------------------------------------------- -const PakLoadedInfo_t* Pak_GetPakInfo(const PakHandle_t pakId) +PakLoadedInfo_t* Pak_GetPakInfo(const PakHandle_t pakId) { - for (int16_t i = 0; i < *g_pLoadedPakCount; ++i) - { - const PakLoadedInfo_t* const info = &g_pLoadedPakInfo[i]; - if (!info) - continue; - - if (info->handle != pakId) - continue; - - return info; - } - - Warning(eDLL_T::RTECH, "%s - Failed to retrieve pak info for handle '%d'\n", __FUNCTION__, pakId); - return nullptr; + return &g_pakGlobals->loadedPaks[pakId & PAK_MAX_HANDLES_MASK]; } //----------------------------------------------------------------------------- @@ -169,9 +170,9 @@ const PakLoadedInfo_t* Pak_GetPakInfo(const PakHandle_t pakId) //----------------------------------------------------------------------------- const PakLoadedInfo_t* Pak_GetPakInfo(const char* const pakName) { - for (int16_t i = 0; i < *g_pLoadedPakCount; ++i) + for (int16_t i = 0; i < g_pakGlobals->loadedPakCount; ++i) { - const PakLoadedInfo_t* const info = &g_pLoadedPakInfo[i]; + const PakLoadedInfo_t* const info = &g_pakGlobals->loadedPaks[i]; if (!info) continue; diff --git a/r5dev/rtech/pak/paktools.h b/r5dev/rtech/pak/paktools.h index 4510d9f6..cd9e5780 100644 --- a/r5dev/rtech/pak/paktools.h +++ b/r5dev/rtech/pak/paktools.h @@ -13,9 +13,11 @@ extern bool Pak_FileOverrideExists(const char* const pakFilePath, char* const ou extern int Pak_FileExists(const char* const pakFilePath); extern const char* Pak_StatusToString(const EPakStatus status); +const char* Pak_DecoderToString(const EPakDecodeMode mode); + extern PakGuid_t Pak_StringToGuid(const char* const string); -extern const PakLoadedInfo_t* Pak_GetPakInfo(const PakHandle_t pakId); +extern PakLoadedInfo_t* Pak_GetPakInfo(const PakHandle_t pakId); extern const PakLoadedInfo_t* Pak_GetPakInfo(const char* const pakName); extern PakPatchDataHeader_t* Pak_GetPatchDataHeader(PakFileHeader_t* const pakHeader);