From bef61249fd516cc08a39d6500e517a9bab957d35 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Wed, 1 Jan 2025 21:40:04 +0100 Subject: [PATCH] RTech: function qualification and symbol naming consistency improvements Use static and const where possible, and rename 'segments' to 'slabs' as this represents a huge buffer in which pages are allocated to, and slab fits this name better. --- src/public/rtech/ipakfile.h | 81 +++++++++++++------------- src/rtech/pak/pakalloc.cpp | 78 ++++++++++++------------- src/rtech/pak/pakalloc.h | 6 +- src/rtech/pak/pakdecode.cpp | 14 ++--- src/rtech/pak/pakparse.cpp | 81 +++++++++++++------------- src/rtech/pak/pakpatch.cpp | 112 ++++++++++++++++-------------------- 6 files changed, 178 insertions(+), 194 deletions(-) diff --git a/src/public/rtech/ipakfile.h b/src/public/rtech/ipakfile.h index 9eff2b5f..e30c1475 100644 --- a/src/public/rtech/ipakfile.h +++ b/src/public/rtech/ipakfile.h @@ -36,11 +36,11 @@ #define PAK_MAX_TRACKED_ASSETS (PAK_MAX_LOADED_ASSETS/2) #define PAK_MAX_TRACKED_ASSETS_MASK (PAK_MAX_TRACKED_ASSETS-1) -// max amount of segments a pak file could have -#define PAK_MAX_SEGMENTS 20 +// max amount of slabs a pak file could have +#define PAK_MAX_SLABS 20 -// max amount of buffers in which segments get copied in -#define PAK_SEGMENT_BUFFER_TYPES 4 +// max amount of buffers in which slabs get copied in +#define PAK_SLAB_BUFFER_TYPES 4 // max amount of streaming files that could be opened per set for a pak, so if a // pak uses more than one set, this number would be used per set @@ -112,7 +112,7 @@ typedef uint64_t PakGuid_t; //----------------------------------------------------------------------------- struct PakPageHeader_s { - uint32_t segmentIdx; + uint32_t slabIndex; uint32_t pageAlignment; uint32_t dataSize; }; @@ -203,9 +203,9 @@ struct PakAssetBinding_s CAlignedMemAlloc* allocator; - unsigned int headerSize; - unsigned int nativeClassSize; // Native class size, for 'material' it would be CMaterialGlue full size. - unsigned int headerAlignment; + uint32_t headerSize; + uint32_t nativeClassSize; // Native class size, for 'material' it would be CMaterialGlue full size. + uint32_t headerAlignment; // the type of this asset bind // NOTE: the asset bind will be stubbed if its 'NONE' in runtime! @@ -229,11 +229,12 @@ struct PakAsset_s uint16_t pageEnd; // the number of remaining dependencies that are yet to be resolved - uint16_t numRemainingDependencies; + int16_t numRemainingDependencies; + uint32_t dependentsIndex; - uint32_t dependenciesIndex; + uint32_t usesIndex; uint32_t dependentsCount; - uint32_t dependenciesCount; + uint32_t usesCount; // size of the asset's header uint32_t headerSize; @@ -313,8 +314,8 @@ public: uint32_t assetCount; const char* fileName; CAlignedMemAlloc* allocator; - PakGuid_t* assetGuids; //size of the array is m_nAssetCount - void* segmentBuffers[PAK_SEGMENT_BUFFER_TYPES]; + PakGuid_t* assetGuids; // size of the array is assetCount + void* slabBuffers[PAK_SLAB_BUFFER_TYPES]; void* guidDestriptors; FILETIME fileTime; PakFile_s* pakFile; @@ -485,20 +486,20 @@ struct PakFileHeader_s // size of the string array containing paths to external streaming files uint16_t streamingFilesBufSize[STREAMING_SET_COUNT]; - // number of segments in this pak; absolute max = PAK_MAX_SEGMENTS - uint16_t virtualSegmentCount; + // number of memory slabs in this pak in which pages get allocated to; absolute max = PAK_MAX_SLABS + uint16_t memSlabCount; // number of memory pages to allocate for this pak uint16_t memPageCount; uint16_t patchIndex; - uint32_t descriptorCount; + uint16_t alignment; - // number of assets in this pak - uint32_t assetCount; - uint32_t guidDescriptorCount; - uint32_t relationsCounts; + uint32_t pointerCount; + uint32_t assetCount; // number of assets in this pak + uint32_t usesCount; + uint32_t dependentsCount; uint8_t unk2[0x10]; @@ -508,26 +509,28 @@ struct PakFileHeader_s uint8_t unk3[0x8]; }; static_assert(sizeof(PakFileHeader_s) == 0x80); -// segment flags -#define SF_HEAD (0) -#define SF_TEMP (1 << 0) // 0x1 -#define SF_CPU (1 << 1) // 0x2 -#define SF_DEV (1 << 8) // 0x80 +// slab flags +#define SF_HEAD (0) +#define SF_CPU (1 << 0) +#define SF_TEMP (1 << 1) +#define SF_SERVER (1 << 5) +#define SF_CLIENT (1 << 6) +#define SF_DEV (1 << 8) -struct PakSegmentHeader_s +struct PakSlabHeader_s { int typeFlags; int dataAlignment; size_t dataSize; }; -struct PakSegmentDescriptor_s +struct PakSlabDescriptor_s { size_t assetTypeCount[PAK_MAX_TRACKED_TYPES]; - int64_t segmentSizes[PAK_MAX_SEGMENTS]; + int64_t slabSizes[PAK_MAX_SLABS]; - size_t segmentSizeForType[PAK_SEGMENT_BUFFER_TYPES]; - int segmentAlignmentForType[PAK_SEGMENT_BUFFER_TYPES]; + size_t slabSizeForType[PAK_SLAB_BUFFER_TYPES]; + int slabAlignmentForType[PAK_SLAB_BUFFER_TYPES]; }; struct PakDecoder_s @@ -687,7 +690,7 @@ struct PakMemoryData_s char* streamingFilePaths[STREAMING_SET_COUNT]; - PakSegmentHeader_s* segmentHeaders; + PakSlabHeader_s* slabHeaders; PakPageHeader_s* pageHeaders; PakPage_u* virtualPointers; @@ -704,7 +707,7 @@ struct PakMemoryData_s int someAssetCount; int numShiftedPointers; - // array of sizes/offsets in the SF_HEAD segment buffer + // array of sizes/offsets in the SF_HEAD slab buffer __int64 unkAssetTypeBindingSizes[PAK_MAX_TRACKED_TYPES]; const char* fileName; @@ -760,7 +763,7 @@ struct PakFile_s inline uint32_t GetPointerCount() const { - return GetHeader().descriptorCount; + return GetHeader().pointerCount; } // --- pages --- @@ -819,17 +822,17 @@ struct PakFile_s return memoryData.memPageBuffers[ptr->index] + ptr->offset; } - // --- segments --- - inline uint16_t GetSegmentCount() const + // --- slabs --- + inline uint16_t GetSlabCount() const { - return GetHeader().virtualSegmentCount; + return GetHeader().memSlabCount; } - inline const PakSegmentHeader_s* GetSegmentHeader(const uint32_t i) const + inline const PakSlabHeader_s* GetSlabHeader(const uint32_t i) const { - assert(i < GetSegmentCount()); + assert(i < GetSlabCount()); - return &memoryData.segmentHeaders[i]; + return &memoryData.slabHeaders[i]; } }; diff --git a/src/rtech/pak/pakalloc.cpp b/src/rtech/pak/pakalloc.cpp index 0533bded..a7c4b9bd 100644 --- a/src/rtech/pak/pakalloc.cpp +++ b/src/rtech/pak/pakalloc.cpp @@ -8,12 +8,12 @@ #include "pakalloc.h" //----------------------------------------------------------------------------- -// aligns the segment headers for each asset type +// aligns the slab headers for each asset type //----------------------------------------------------------------------------- -void Pak_AlignSegmentHeaders(PakFile_s* const pak, PakSegmentDescriptor_s* const desc) +void Pak_AlignSlabHeaders(PakFile_s* const pak, PakSlabDescriptor_s* const desc) { uint64_t headersSize = 0; - uint8_t headerSegmentAlignment = static_cast(desc->segmentAlignmentForType[SF_HEAD]); + uint32_t slabHeaderAlignment = desc->slabAlignmentForType[SF_HEAD]; for (uint8_t i = 0; i < PAK_MAX_TRACKED_TYPES; ++i) { @@ -21,19 +21,16 @@ void Pak_AlignSegmentHeaders(PakFile_s* const pak, PakSegmentDescriptor_s* const if (desc->assetTypeCount[i]) { - // asset header alignment really shouldn't be above 255 - // if this needs raising, headerSegmentAlignment should be made wider - assert(binding.headerAlignment <= UINT8_MAX); - + assert(binding.headerAlignment > 0 && IsPowerOfTwo(binding.headerAlignment)); const size_t alignedSize = ALIGN_VALUE(headersSize, static_cast(binding.headerAlignment)); pak->memoryData.unkAssetTypeBindingSizes[i] = alignedSize; headersSize = alignedSize + (desc->assetTypeCount[i] * binding.nativeClassSize); - desc->segmentSizeForType[SF_HEAD] = headersSize; + desc->slabSizeForType[SF_HEAD] = headersSize; - headerSegmentAlignment = Max(headerSegmentAlignment, static_cast(binding.headerAlignment)); - desc->segmentAlignmentForType[SF_HEAD] = headerSegmentAlignment; + slabHeaderAlignment = Max(slabHeaderAlignment, binding.headerAlignment); + desc->slabAlignmentForType[SF_HEAD] = slabHeaderAlignment; } } } @@ -41,69 +38,66 @@ void Pak_AlignSegmentHeaders(PakFile_s* const pak, PakSegmentDescriptor_s* const //----------------------------------------------------------------------------- // aligns each individual non-header segment //----------------------------------------------------------------------------- -void Pak_AlignSegments(PakFile_s* const pak, PakSegmentDescriptor_s* const desc) +void Pak_AlignSlabData(PakFile_s* const pak, PakSlabDescriptor_s* const desc) { - for (uint16_t i = 0; i < pak->GetSegmentCount(); ++i) + for (uint16_t i = 0; i < pak->GetSlabCount(); ++i) { - const PakSegmentHeader_s* const segHeader = pak->GetSegmentHeader(i); + const PakSlabHeader_s* const slabHeader = pak->GetSlabHeader(i); + const uint8_t slabType = slabHeader->typeFlags & (SF_CPU | SF_TEMP); - const uint8_t segmentType = segHeader->typeFlags & (SF_TEMP | SF_CPU); - - if (segmentType != SF_HEAD) // if not a header segment + if (slabType != SF_HEAD) // if not a header slab { // should this be a hard error on release? - // segment alignment must not be 0 and must be a power of two - assert(segHeader->dataAlignment > 0 && IsPowerOfTwo(segHeader->dataAlignment)); + // slab alignment must not be 0 and must be a power of two + assert(slabHeader->dataAlignment > 0 && IsPowerOfTwo(slabHeader->dataAlignment)); + const size_t alignedSlabSize = ALIGN_VALUE(desc->slabSizeForType[slabType], static_cast(slabHeader->dataAlignment)); - const size_t alignedSegmentSize = ALIGN_VALUE(desc->segmentSizeForType[segmentType], static_cast(segHeader->dataAlignment)); - //const size_t sizeAligned = ~(m_align - 1) & (m_align - 1 + segmentSizeForType[segmentType]); + desc->slabSizes[i] = alignedSlabSize; + desc->slabSizeForType[slabType] = alignedSlabSize + slabHeader->dataSize; - desc->segmentSizes[i] = alignedSegmentSize; - desc->segmentSizeForType[segmentType] = alignedSegmentSize + segHeader->dataSize; - - // check if this segment's alignment is higher than the previous highest for this type - // if so, increase the alignment to accommodate this segment - desc->segmentAlignmentForType[segmentType] = Max(desc->segmentAlignmentForType[segmentType], segHeader->dataAlignment); + // check if this slab's alignment is higher than the previous highest for this type + // if so, increase the alignment to accommodate this slab + desc->slabAlignmentForType[slabType] = Max(desc->slabAlignmentForType[slabType], slabHeader->dataAlignment); } } } //----------------------------------------------------------------------------- -// copy's pages into pre-allocated and aligned segments +// copy's pages into pre-allocated and aligned slabs //----------------------------------------------------------------------------- -void Pak_CopyPagesToSegments(PakFile_s* const pak, PakLoadedInfo_s* const loadedInfo, PakSegmentDescriptor_s* const desc) +void Pak_CopyPagesToSlabs(PakFile_s* const pak, PakLoadedInfo_s* const loadedInfo, PakSlabDescriptor_s* const desc) { for (uint32_t i = 0; i < pak->GetPageCount(); ++i) { const PakPageHeader_s* const pageHeader = pak->GetPageHeader(i); - const uint32_t segmentIndex = pageHeader->segmentIdx; + const uint32_t slabIndex = pageHeader->slabIndex; - const PakSegmentHeader_s* const segHeader = pak->GetSegmentHeader(segmentIndex); - const int typeFlags = segHeader->typeFlags; + const PakSlabHeader_s* const slabHeader = pak->GetSlabHeader(slabIndex); + const int typeFlags = slabHeader->typeFlags; // check if header page - if ((typeFlags & (SF_TEMP | SF_CPU)) != 0) + if ((typeFlags & (SF_CPU | SF_TEMP)) != 0) { - // align the segment's current size to the alignment of the new page to get copied in + // align the slab's current size to the alignment of the new page to get copied in // this ensures that the location holding the page is aligned as required // - // since the segment will always have alignment equal to or greater than the page, and that alignment will always be a power of 2 - // the page does not have to be aligned to the same alignment as the segment, as aligning it to its own alignment is sufficient as long as + // since the slab will always have alignment equal to or greater than the page, and that alignment will always be a power of 2 + // the page does not have to be aligned to the same alignment as the slab, as aligning it to its own alignment is sufficient as long as // every subsequent page does the same thing - const size_t alignedSegmentSize = ALIGN_VALUE(desc->segmentSizes[segmentIndex], static_cast(pageHeader->pageAlignment)); + const size_t alignedSlabSize = ALIGN_VALUE(desc->slabSizes[slabIndex], static_cast(pageHeader->pageAlignment)); - // get a pointer to the newly aligned location within the segment for this page - pak->memoryData.memPageBuffers[i] = reinterpret_cast(loadedInfo->segmentBuffers[typeFlags & (SF_TEMP | SF_CPU)]) + alignedSegmentSize; + // get a pointer to the newly aligned location within the slab for this page + pak->memoryData.memPageBuffers[i] = reinterpret_cast(loadedInfo->slabBuffers[typeFlags & (SF_CPU | SF_TEMP)]) + alignedSlabSize; - // update the segment size to reflect the new alignment and page size - desc->segmentSizes[segmentIndex] = alignedSegmentSize + pak->memoryData.pageHeaders[i].dataSize; + // update the slab size to reflect the new alignment and page size + desc->slabSizes[slabIndex] = alignedSlabSize + pak->memoryData.pageHeaders[i].dataSize; } else { - // all headers go into one segment and are dealt with separately in Pak_ProcessPakFile + // all headers go into one slab and are dealt with separately in Pak_ProcessPakFile // since headers must be copied individually into a buffer that is big enough for the "native class" version of the header // instead of just the file version - pak->memoryData.memPageBuffers[i] = reinterpret_cast(loadedInfo->segmentBuffers[SF_HEAD]); + pak->memoryData.memPageBuffers[i] = reinterpret_cast(loadedInfo->slabBuffers[SF_HEAD]); } } } diff --git a/src/rtech/pak/pakalloc.h b/src/rtech/pak/pakalloc.h index 58d06382..68d71a71 100644 --- a/src/rtech/pak/pakalloc.h +++ b/src/rtech/pak/pakalloc.h @@ -2,9 +2,9 @@ #define RTECH_PAKALLOC_H #include "rtech/ipakfile.h" -extern void Pak_AlignSegmentHeaders(PakFile_s* const pak, PakSegmentDescriptor_s* const desc); -extern void Pak_AlignSegments(PakFile_s* const pak, PakSegmentDescriptor_s* const desc); -extern void Pak_CopyPagesToSegments(PakFile_s* const pak, PakLoadedInfo_s* const loadedInfo, PakSegmentDescriptor_s* const desc); +extern void Pak_AlignSlabHeaders(PakFile_s* const pak, PakSlabDescriptor_s* const desc); +extern void Pak_AlignSlabData(PakFile_s* const pak, PakSlabDescriptor_s* const desc); +extern void Pak_CopyPagesToSlabs(PakFile_s* const pak, PakLoadedInfo_s* const loadedInfo, PakSlabDescriptor_s* const desc); // something with sorting pages? inline void (*sub_140442740)(PakAsset_s** assetEntries, PakAsset_s** assetEntry, __int64 idx, PakFile_s* pak); diff --git a/src/rtech/pak/pakdecode.cpp b/src/rtech/pak/pakdecode.cpp index c9075521..847091f6 100644 --- a/src/rtech/pak/pakdecode.cpp +++ b/src/rtech/pak/pakdecode.cpp @@ -135,7 +135,7 @@ static const unsigned char /*141313180*/ s_defaultDecoderLUT[] = //----------------------------------------------------------------------------- // checks if we have enough output buffer room to decode the data stream //----------------------------------------------------------------------------- -bool Pak_HasEnoughDecodeBufferAvailable(PakDecoder_s* const decoder, const size_t outLen) +static bool Pak_HasEnoughDecodeBufferAvailable(PakDecoder_s* const decoder, const size_t outLen) { // make sure caller has copied all data out the ring buffer first before // overwriting it with new decoded data @@ -146,7 +146,7 @@ bool Pak_HasEnoughDecodeBufferAvailable(PakDecoder_s* const decoder, const size_ //----------------------------------------------------------------------------- // checks if we have enough source data streamed to decode the next block //----------------------------------------------------------------------------- -bool Pak_HasEnoughStreamedDataForDecode(PakDecoder_s* const decoder, const size_t inLen) +static bool Pak_HasEnoughStreamedDataForDecode(PakDecoder_s* const decoder, const size_t inLen) { // the decoder needs at least this amount of input data streamed in order // to decode the rest of the pak file, as this is where reading has stopped @@ -160,7 +160,7 @@ bool Pak_HasEnoughStreamedDataForDecode(PakDecoder_s* const decoder, const size_ // gets the frame for the data in the ring buffer, the frame returned is always // ending to the end of the ring buffer, or the end of the data itself //----------------------------------------------------------------------------- -PakRingBufferFrame_s Pak_DetermineRingBufferFrame(const uint64_t bufMask, const size_t seekPos, const size_t dataLen) +static PakRingBufferFrame_s Pak_DetermineRingBufferFrame(const uint64_t bufMask, const size_t seekPos, const size_t dataLen) { PakRingBufferFrame_s ring; ring.bufIndex = seekPos & bufMask; @@ -178,7 +178,7 @@ PakRingBufferFrame_s Pak_DetermineRingBufferFrame(const uint64_t bufMask, const //----------------------------------------------------------------------------- // initializes the RTech decoder //----------------------------------------------------------------------------- -size_t Pak_RTechDecoderInit(PakDecoder_s* const decoder, const uint8_t* const fileBuffer, +static size_t Pak_RTechDecoderInit(PakDecoder_s* const decoder, const uint8_t* const fileBuffer, const uint64_t inputMask, const size_t dataSize, const size_t dataOffset, const size_t headerSize) { uint64_t frameHeader = *(_QWORD*)((inputMask & (dataOffset + headerSize)) + fileBuffer); @@ -243,7 +243,7 @@ size_t Pak_RTechDecoderInit(PakDecoder_s* const decoder, const uint8_t* const fi //----------------------------------------------------------------------------- // decodes the RTech data stream up to available buffer or data //----------------------------------------------------------------------------- -bool Pak_RTechStreamDecode(PakDecoder_s* const decoder, const size_t inLen, const size_t outLen) +static bool Pak_RTechStreamDecode(PakDecoder_s* const decoder, const size_t inLen, const size_t outLen) { bool result; // al uint64_t outBufBytePos; // r15 @@ -570,7 +570,7 @@ LABEL_69: //----------------------------------------------------------------------------- // initializes the ZStd decoder //----------------------------------------------------------------------------- -size_t Pak_ZStdDecoderInit(PakDecoder_s* const decoder, const uint8_t* frameHeader, +static size_t Pak_ZStdDecoderInit(PakDecoder_s* const decoder, const uint8_t* frameHeader, const size_t dataSize, const size_t headerSize) { ZSTD_DStream* const dctx = ZSTD_createDStream(); @@ -608,7 +608,7 @@ size_t Pak_ZStdDecoderInit(PakDecoder_s* const decoder, const uint8_t* frameHead // decodes the ZStd data stream up to available buffer or data, whichever ends // first //----------------------------------------------------------------------------- -bool Pak_ZStdStreamDecode(PakDecoder_s* const decoder, const PakRingBufferFrame_s& outFrame, const PakRingBufferFrame_s& inFrame) +static bool Pak_ZStdStreamDecode(PakDecoder_s* const decoder, const PakRingBufferFrame_s& outFrame, const PakRingBufferFrame_s& inFrame) { ZSTD_outBuffer outBuffer = { &decoder->outputBuf[outFrame.bufIndex], diff --git a/src/rtech/pak/pakparse.cpp b/src/rtech/pak/pakparse.cpp index fb578e32..2f18eee8 100644 --- a/src/rtech/pak/pakparse.cpp +++ b/src/rtech/pak/pakparse.cpp @@ -49,16 +49,16 @@ static bool Pak_ResolveAssetDependency(const PakFile_s* const pak, PakGuid_t cur //----------------------------------------------------------------------------- // resolve guid relations for asset //----------------------------------------------------------------------------- -void Pak_ResolveAssetRelations(PakFile_s* const pak, const PakAsset_s* const asset) +static void Pak_ResolveAssetRelations(PakFile_s* const pak, const PakAsset_s* const asset) { - PakPage_u* const pageDescriptors = &pak->memoryData.pageDescriptors[asset->dependenciesIndex]; + PakPage_u* const pageDescriptors = &pak->memoryData.pageDescriptors[asset->usesIndex]; uint32_t* const guidDestriptors = (uint32_t*)g_pakGlobals->loadedPaks[pak->memoryData.pakId & PAK_MAX_LOADED_PAKS_MASK].guidDestriptors; if (pak_debugrelations.GetBool()) Msg(eDLL_T::RTECH, "Resolving relations for asset: '0x%-16llX', dependencies: %-4u; in pak '%s'\n", - asset->guid, asset->dependenciesCount, pak->memoryData.fileName); + asset->guid, asset->usesCount, pak->memoryData.fileName); - for (uint32_t i = 0; i < asset->dependenciesCount; i++) + for (uint32_t i = 0; i < asset->usesCount; i++) { void** const pCurrentGuid = reinterpret_cast(pak->memoryData.memPageBuffers[pageDescriptors[i].index] + pageDescriptors[i].offset); @@ -93,7 +93,7 @@ void Pak_ResolveAssetRelations(PakFile_s* const pak, const PakAsset_s* const ass "pak: '%s'\n" "asset: '0x%llX'\n" "target: '0x%llX'\n", - i, asset->dependenciesCount, + i, asset->usesCount, pak->memoryData.fileName, asset->guid, targetGuid); @@ -112,7 +112,7 @@ void Pak_ResolveAssetRelations(PakFile_s* const pak, const PakAsset_s* const ass } } -uint32_t Pak_ProcessRemainingPagePointers(PakFile_s* const pak) +static uint32_t Pak_ProcessRemainingPagePointers(PakFile_s* const pak) { uint32_t processedPointers = 0; @@ -134,7 +134,7 @@ uint32_t Pak_ProcessRemainingPagePointers(PakFile_s* const pak) return processedPointers; } -void Pak_RunAssetLoadingJobs(PakFile_s* const pak) +static void Pak_RunAssetLoadingJobs(PakFile_s* const pak) { pak->numProcessedPointers = Pak_ProcessRemainingPagePointers(pak); @@ -167,7 +167,7 @@ void Pak_RunAssetLoadingJobs(PakFile_s* const pak) } else { - if (_InterlockedExchangeAdd16((volatile signed __int16*)&pakAsset->numRemainingDependencies, 0xFFFFu) == 1) + if (_InterlockedExchangeAdd16(&pakAsset->numRemainingDependencies, -1) == 1) Pak_ProcessAssetRelationsAndResolveDependencies(pak, pakAsset, currentAsset, assetBind); _InterlockedDecrement16(&g_pakGlobals->numAssetLoadJobs); @@ -191,7 +191,7 @@ void Pak_RunAssetLoadingJobs(PakFile_s* const pak) //----------------------------------------------------------------------------- // load user-requested pak files on-demand //----------------------------------------------------------------------------- -PakHandle_t Pak_LoadAsync(const char* const fileName, CAlignedMemAlloc* const allocator, const int logChannel, const bool bUnk) +static PakHandle_t Pak_LoadAsync(const char* const fileName, CAlignedMemAlloc* const allocator, const int logChannel, const bool bUnk) { if (!Pak_FileExists(fileName)) { @@ -216,7 +216,7 @@ PakHandle_t Pak_LoadAsync(const char* const fileName, CAlignedMemAlloc* const al //----------------------------------------------------------------------------- // unloads loaded pak files //----------------------------------------------------------------------------- -void Pak_UnloadAsync(const PakHandle_t handle) +static void Pak_UnloadAsync(const PakHandle_t handle) { const PakLoadedInfo_s* const pakInfo = Pak_GetPakInfo(handle); @@ -238,9 +238,8 @@ static const int s_patchCmdToBytesToProcess[] = { CMD_INVALID, CMD_INVALID, CMD_ #undef CMD_INVALID //---------------------------------------------------------------------------------- // loads and processes a pak file (handles decompression and patching) -// TODO: !!! FINISH REBUILD !!! //---------------------------------------------------------------------------------- -bool Pak_ProcessPakFile(PakFile_s* const pak) +static bool Pak_ProcessPakFile(PakFile_s* const pak) { PakFileStream_s* const fileStream = &pak->fileStream; PakMemoryData_s* const memoryData = &pak->memoryData; @@ -522,9 +521,9 @@ bool Pak_ProcessPakFile(PakFile_s* const pak) return memoryData->patchSrcSize == 0; } -// sets patch variables for copying the next unprocessed page into the relevant segment buffer +// sets patch variables for copying the next unprocessed page into the relevant slab buffer // if this is a header page, fetch info from the next unprocessed asset and copy over the asset's header -bool Pak_PrepareNextPageForPatching(PakLoadedInfo_s* const loadedInfo, PakFile_s* const pak) +static bool Pak_PrepareNextPageForPatching(PakLoadedInfo_s* const loadedInfo, PakFile_s* const pak) { Pak_RunAssetLoadingJobs(pak); @@ -542,7 +541,7 @@ bool Pak_PrepareNextPageForPatching(PakLoadedInfo_s* const loadedInfo, PakFile_s : highestProcessedPageIdx - pak->GetPageCount(); const PakPageHeader_s* const nextMemPageHeader = &pak->memoryData.pageHeaders[currentPageIndex]; - if ((pak->memoryData.segmentHeaders[nextMemPageHeader->segmentIdx].typeFlags & (SF_TEMP | SF_CPU)) != 0) + if ((pak->memoryData.slabHeaders[nextMemPageHeader->slabIndex].typeFlags & (SF_CPU | SF_TEMP)) != 0) { pak->memoryData.patchSrcSize = nextMemPageHeader->dataSize; pak->memoryData.patchDstPtr = reinterpret_cast(pak->memoryData.memPageBuffers[currentPageIndex]); @@ -556,13 +555,13 @@ bool Pak_PrepareNextPageForPatching(PakLoadedInfo_s* const loadedInfo, PakFile_s pak->memoryData.patchSrcSize = pakAsset->headerSize; const int assetTypeIdx = pakAsset->HashTableIndexForAssetType(); - pak->memoryData.patchDstPtr = reinterpret_cast(loadedInfo->segmentBuffers[0]) + pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx]; + pak->memoryData.patchDstPtr = reinterpret_cast(loadedInfo->slabBuffers[0]) + pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx]; pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx] += g_pakGlobals->assetBindings[assetTypeIdx].nativeClassSize; return true; } -bool Pak_ProcessAssets(PakLoadedInfo_s* const loadedInfo) +static bool Pak_ProcessAssets(PakLoadedInfo_s* const loadedInfo) { PakFile_s* const pak = loadedInfo->pakFile; @@ -593,8 +592,8 @@ bool Pak_ProcessAssets(PakLoadedInfo_s* const loadedInfo) if (v4 >= pageCount) shiftedPageIndex -= pageCount; - // if "temp_" segment - if ((pak->memoryData.segmentHeaders[pak->memoryData.pageHeaders[shiftedPageIndex].segmentIdx].typeFlags & (SF_TEMP | SF_CPU)) != 0) + // if "temp_" slab + if ((pak->memoryData.slabHeaders[pak->memoryData.pageHeaders[shiftedPageIndex].slabIndex].typeFlags & (SF_CPU | SF_TEMP)) != 0) { if (Pak_PrepareNextPageForPatching(loadedInfo, pak)) continue; @@ -606,12 +605,12 @@ bool Pak_ProcessAssets(PakLoadedInfo_s* const loadedInfo) const uint32_t headPageOffset = asset->headPtr.offset; char* const v8 = pak->memoryData.patchDstPtr - asset->headerSize; - const uint32_t newOffsetFromSegmentBufferToHeader = LODWORD(pak->memoryData.patchDstPtr) + const uint32_t newOffsetFromSlabBufferToHeader = LODWORD(pak->memoryData.patchDstPtr) - asset->headerSize - - LODWORD(loadedInfo->segmentBuffers[0]); - asset->headPtr.offset = newOffsetFromSegmentBufferToHeader; + - LODWORD(loadedInfo->slabBuffers[0]); + asset->headPtr.offset = newOffsetFromSlabBufferToHeader; - const uint32_t offsetSize = newOffsetFromSegmentBufferToHeader - headPageOffset; + const uint32_t offsetSize = newOffsetFromSlabBufferToHeader - headPageOffset; for (uint32_t i = pak->memoryData.numShiftedPointers; i < pak->GetPointerCount(); pak->memoryData.numShiftedPointers = i) { @@ -638,9 +637,9 @@ bool Pak_ProcessAssets(PakLoadedInfo_s* const loadedInfo) i = pak->memoryData.numShiftedPointers + 1; } - for (uint32_t j = 0; j < asset->dependenciesCount; ++j) + for (uint32_t j = 0; j < asset->usesCount; ++j) { - PakPage_u* const descriptor = &pak->memoryData.pageDescriptors[asset->dependenciesIndex + j]; + PakPage_u* const descriptor = &pak->memoryData.pageDescriptors[asset->usesIndex + j]; if (descriptor->index == shiftedPageIndex) descriptor->offset += offsetSize; @@ -655,7 +654,7 @@ bool Pak_ProcessAssets(PakLoadedInfo_s* const loadedInfo) pak->memoryData.patchSrcSize = v17->headerSize; const uint8_t assetTypeIdx = v17->HashTableIndexForAssetType(); - pak->memoryData.patchDstPtr = reinterpret_cast(loadedInfo->segmentBuffers[0]) + pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx]; + pak->memoryData.patchDstPtr = reinterpret_cast(loadedInfo->slabBuffers[0]) + pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx]; pak->memoryData.unkAssetTypeBindingSizes[assetTypeIdx] += g_pakGlobals->assetBindings[assetTypeIdx].nativeClassSize; } @@ -735,7 +734,7 @@ bool Pak_ProcessAssets(PakLoadedInfo_s* const loadedInfo) return true; } -void Pak_StubInvalidAssetBinds(PakFile_s* const pak, PakSegmentDescriptor_s* const desc) +static void Pak_StubInvalidAssetBinds(PakFile_s* const pak, PakSlabDescriptor_s* const desc) { for (uint32_t i = 0; i < pak->GetAssetCount(); ++i) { @@ -781,16 +780,16 @@ void Pak_StubInvalidAssetBinds(PakFile_s* const pak, PakSegmentDescriptor_s* con } } -bool Pak_StartLoadingPak(PakLoadedInfo_s* const loadedInfo) +static bool Pak_StartLoadingPak(PakLoadedInfo_s* const loadedInfo) { PakFile_s* const pakFile = loadedInfo->pakFile; if (pakFile->memoryData.patchSrcSize && !Pak_ProcessPakFile(pakFile)) return false; - PakSegmentDescriptor_s pakDescriptor = {}; + PakSlabDescriptor_s slabDesc = {}; - Pak_StubInvalidAssetBinds(pakFile, &pakDescriptor); + Pak_StubInvalidAssetBinds(pakFile, &slabDesc); const uint32_t numAssets = pakFile->GetAssetCount(); @@ -799,33 +798,33 @@ bool Pak_StartLoadingPak(PakLoadedInfo_s* const loadedInfo) sub_140442740(pakFile->memoryData.ppAssetEntries, &pakFile->memoryData.ppAssetEntries[numAssets], numAssets, pakFile); - // pak must have no more than PAK_MAX_SEGMENTS segments as otherwise we will overrun the above "segmentSizes" array + // pak must have no more than PAK_MAX_SLABS slabs as otherwise we will overrun the above "slabSizes" array // and write to arbitrary locations on the stack - if (pakFile->GetSegmentCount() > PAK_MAX_SEGMENTS) + if (pakFile->GetSlabCount() > PAK_MAX_SLABS) { - Error(eDLL_T::RTECH, EXIT_FAILURE, "Too many segments in pakfile '%s'. Max %i, found %i.\n", pakFile->GetName(), PAK_MAX_SEGMENTS, pakFile->GetSegmentCount()); + Error(eDLL_T::RTECH, EXIT_FAILURE, "Too many slabs in pakfile '%s'. Max %hu, found %hu.\n", pakFile->GetName(), PAK_MAX_SLABS, pakFile->GetSlabCount()); return false; } - Pak_AlignSegmentHeaders(pakFile, &pakDescriptor); - Pak_AlignSegments(pakFile, &pakDescriptor); + Pak_AlignSlabHeaders(pakFile, &slabDesc); + Pak_AlignSlabData(pakFile, &slabDesc); - // allocate segment buffers with predetermined alignments; pages will be + // allocate slab buffers with predetermined alignments; pages will be // copied into here - for (int8_t i = 0; i < PAK_SEGMENT_BUFFER_TYPES; ++i) + for (int8_t i = 0; i < PAK_SLAB_BUFFER_TYPES; ++i) { - if (pakDescriptor.segmentSizeForType[i]) - loadedInfo->segmentBuffers[i] = AlignedMemAlloc()->Alloc(pakDescriptor.segmentSizeForType[i], pakDescriptor.segmentAlignmentForType[i]); + if (slabDesc.slabSizeForType[i]) + loadedInfo->slabBuffers[i] = AlignedMemAlloc()->Alloc(slabDesc.slabSizeForType[i], slabDesc.slabAlignmentForType[i]); } - Pak_CopyPagesToSegments(pakFile, loadedInfo, &pakDescriptor); + Pak_CopyPagesToSlabs(pakFile, loadedInfo, &slabDesc); const PakFileHeader_s& pakHdr = pakFile->GetHeader(); if (Pak_StreamingEnabled()) Pak_LoadStreamingData(loadedInfo); - const __int64 v106 = pakHdr.descriptorCount + 2 * (pakHdr.patchIndex + pakHdr.assetCount + 4ull * pakHdr.assetCount + pakHdr.virtualSegmentCount); + const __int64 v106 = pakHdr.pointerCount + 2 * (pakHdr.patchIndex + pakHdr.assetCount + 4ull * pakHdr.assetCount + pakHdr.memSlabCount); const __int64 patchDestOffset = pakHdr.GetTotalHeaderSize() + 2 * (pakHdr.patchIndex + 6ull * pakHdr.memPageCount + 4 * v106); pakFile->dword14 = 1; diff --git a/src/rtech/pak/pakpatch.cpp b/src/rtech/pak/pakpatch.cpp index 5cf8b57a..b05edd8e 100644 --- a/src/rtech/pak/pakpatch.cpp +++ b/src/rtech/pak/pakpatch.cpp @@ -6,7 +6,7 @@ #include "rtech/ipakfile.h" #include "pakpatch.h" -bool PATCH_CMD_0(PakFile_s* const pak, size_t* const numAvailableBytes) +static bool PATCH_CMD_0(PakFile_s* const pak, size_t* const numAvailableBytes) { unsigned __int64 m_numBytesToProcess_maybe; // r9 unsigned __int64 v4; // rdi @@ -83,65 +83,55 @@ bool PATCH_CMD_0(PakFile_s* const pak, size_t* const numAvailableBytes) return pak->memoryData.numPatchBytesToProcess == 0; } -bool PATCH_CMD_1(PakFile_s* const pak, size_t* const numAvailableBytes) +static bool PATCH_CMD_1(PakFile_s* const pak, size_t* const pNumBytesAvailable) { - unsigned __int64 m_numBytesToProcess_maybe; // r8 - size_t v3; // r9 - uint64_t m_processedPatchedDataSize; // rax + const size_t numBytesToProcess = pak->memoryData.numPatchBytesToProcess; + const size_t numBytesAvailable = *pNumBytesAvailable; + const size_t processedPatchedDataSize = pak->memoryData.processedPatchedDataSize; - m_numBytesToProcess_maybe = pak->memoryData.numPatchBytesToProcess; - v3 = *numAvailableBytes; - m_processedPatchedDataSize = pak->memoryData.processedPatchedDataSize; - - if (*numAvailableBytes > m_numBytesToProcess_maybe) + if (*pNumBytesAvailable > numBytesToProcess) { - pak->memoryData.numPatchBytesToProcess = 0i64; - pak->memoryData.processedPatchedDataSize += m_numBytesToProcess_maybe; - *numAvailableBytes = v3 - m_numBytesToProcess_maybe; + pak->memoryData.numPatchBytesToProcess = 0ull; + pak->memoryData.processedPatchedDataSize += numBytesToProcess; + *pNumBytesAvailable = numBytesAvailable - numBytesToProcess; return true; } else { - pak->memoryData.processedPatchedDataSize += v3; - pak->memoryData.numPatchBytesToProcess -= v3; - *numAvailableBytes = NULL; + pak->memoryData.processedPatchedDataSize += numBytesAvailable; + pak->memoryData.numPatchBytesToProcess -= numBytesAvailable; + *pNumBytesAvailable = NULL; return false; } } -bool PATCH_CMD_2(PakFile_s* const pak, size_t* const numAvailableBytes) +static bool PATCH_CMD_2(PakFile_s* const pak, size_t* const pNumBytesAvailable) { - NOTE_UNUSED(numAvailableBytes); + NOTE_UNUSED(pNumBytesAvailable); - unsigned __int64 m_numBytesToProcess_maybe; - unsigned __int64 v3; - const char* m_patchDataPtr; - - m_numBytesToProcess_maybe = pak->memoryData.numPatchBytesToProcess; - v3 = pak->memoryData.field_2A8; + size_t numBytesToProcess = pak->memoryData.numPatchBytesToProcess; + const size_t v3 = pak->memoryData.field_2A8; if (v3) { - m_patchDataPtr = pak->memoryData.patchDataPtr; - - if (m_numBytesToProcess_maybe <= v3) + if (numBytesToProcess <= v3) { - pak->memoryData.numPatchBytesToProcess = 0i64; - pak->memoryData.patchDataPtr += m_numBytesToProcess_maybe; - pak->memoryData.field_2A8 = v3 - m_numBytesToProcess_maybe; + pak->memoryData.numPatchBytesToProcess = 0ull; + pak->memoryData.patchDataPtr += numBytesToProcess; + pak->memoryData.field_2A8 = v3 - numBytesToProcess; return true; } pak->memoryData.field_2A8 = 0i64; - m_numBytesToProcess_maybe -= v3; + numBytesToProcess -= v3; pak->memoryData.patchDataPtr += v3; - pak->memoryData.numPatchBytesToProcess = m_numBytesToProcess_maybe; + pak->memoryData.numPatchBytesToProcess = numBytesToProcess; } - const size_t patchSrcSize = min(m_numBytesToProcess_maybe, pak->memoryData.patchSrcSize); + const size_t patchSrcSize = Min(numBytesToProcess, pak->memoryData.patchSrcSize); memcpy(pak->memoryData.patchDstPtr, pak->memoryData.patchDataPtr, patchSrcSize); @@ -153,13 +143,10 @@ bool PATCH_CMD_2(PakFile_s* const pak, size_t* const numAvailableBytes) return pak->memoryData.numPatchBytesToProcess == 0; } -bool PATCH_CMD_3(PakFile_s* const pak, size_t* const numAvailableBytes) +static bool PATCH_CMD_3(PakFile_s* const pak, size_t* const pNumBytesAvailable) { - size_t patchSrcSize = pak->memoryData.patchSrcSize; - - size_t v9 = min(*numAvailableBytes, pak->memoryData.numPatchBytesToProcess); - - patchSrcSize = min(v9, patchSrcSize); + const size_t numBytesLeft = Min(*pNumBytesAvailable, pak->memoryData.numPatchBytesToProcess); + const size_t patchSrcSize = Min(numBytesLeft, pak->memoryData.patchSrcSize); memcpy(pak->memoryData.patchDstPtr, pak->memoryData.patchDataPtr, patchSrcSize); pak->memoryData.patchDataPtr += patchSrcSize; @@ -167,15 +154,16 @@ bool PATCH_CMD_3(PakFile_s* const pak, size_t* const numAvailableBytes) pak->memoryData.patchSrcSize -= patchSrcSize; pak->memoryData.patchDstPtr += patchSrcSize; pak->memoryData.numPatchBytesToProcess -= patchSrcSize; - *numAvailableBytes = *numAvailableBytes - patchSrcSize; + *pNumBytesAvailable = *pNumBytesAvailable - patchSrcSize; return pak->memoryData.numPatchBytesToProcess == 0; } -bool PATCH_CMD_4_5(PakFile_s* const pak, size_t* const numAvailableBytes) +static bool PATCH_CMD_4_5(PakFile_s* const pak, size_t* const pNumBytesAvailable) { - const size_t v2 = *numAvailableBytes; - if (!v2) + const size_t numBytesAvailable = *pNumBytesAvailable; + + if (!numBytesAvailable) return false; *pak->memoryData.patchDstPtr = *(_BYTE*)pak->memoryData.patchDataPtr++; @@ -183,29 +171,29 @@ bool PATCH_CMD_4_5(PakFile_s* const pak, size_t* const numAvailableBytes) --pak->memoryData.patchSrcSize; ++pak->memoryData.patchDstPtr; pak->memoryData.patchFunc = PATCH_CMD_0; - *numAvailableBytes = v2 - 1; + *pNumBytesAvailable = numBytesAvailable - 1; - return PATCH_CMD_0(pak, numAvailableBytes); + return PATCH_CMD_0(pak, pNumBytesAvailable); } -bool PATCH_CMD_6(PakFile_s* const pak, size_t* const numAvailableBytes) +static bool PATCH_CMD_6(PakFile_s* const pak, size_t* const pNumBytesAvailable) { - const size_t v2 = *numAvailableBytes; - size_t v3 = 2; + const size_t numBytesAvailable = *pNumBytesAvailable; + size_t numBytesToSkip = 2; - if (*numAvailableBytes < 2) + if (*pNumBytesAvailable < 2) { - if (!*numAvailableBytes) + if (!*pNumBytesAvailable) return false; - v3 = *numAvailableBytes; + numBytesToSkip = *pNumBytesAvailable; } const void* const patchDataPtr = (const void*)pak->memoryData.patchDataPtr; const size_t patchSrcSize = pak->memoryData.patchSrcSize; char* const patchDstPtr = pak->memoryData.patchDstPtr; - if (v3 > patchSrcSize) + if (numBytesToSkip > patchSrcSize) { memcpy(patchDstPtr, patchDataPtr, patchSrcSize); pak->memoryData.patchDataPtr += patchSrcSize; @@ -213,26 +201,26 @@ bool PATCH_CMD_6(PakFile_s* const pak, size_t* const numAvailableBytes) pak->memoryData.patchSrcSize -= patchSrcSize; pak->memoryData.patchDstPtr += patchSrcSize; pak->memoryData.patchFunc = PATCH_CMD_4_5; - *numAvailableBytes = v2 - patchSrcSize; + *pNumBytesAvailable = numBytesAvailable - patchSrcSize; } else { - memcpy(patchDstPtr, patchDataPtr, v3); - pak->memoryData.patchDataPtr += v3; - pak->memoryData.processedPatchedDataSize += v3; - pak->memoryData.patchSrcSize -= v3; - pak->memoryData.patchDstPtr += v3; + memcpy(patchDstPtr, patchDataPtr, numBytesToSkip); + pak->memoryData.patchDataPtr += numBytesToSkip; + pak->memoryData.processedPatchedDataSize += numBytesToSkip; + pak->memoryData.patchSrcSize -= numBytesToSkip; + pak->memoryData.patchDstPtr += numBytesToSkip; - if (v2 >= 2) + if (numBytesAvailable >= 2) { pak->memoryData.patchFunc = PATCH_CMD_0; - *numAvailableBytes = v2 - v3; + *pNumBytesAvailable = numBytesAvailable - numBytesToSkip; - return PATCH_CMD_0(pak, numAvailableBytes); + return PATCH_CMD_0(pak, pNumBytesAvailable); } pak->memoryData.patchFunc = PATCH_CMD_4_5; - *numAvailableBytes = NULL; + *pNumBytesAvailable = NULL; } return false;