From 42d12a644ed4cb3249ee0b5804f5e6e9ddfe5c6c Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Wed, 5 Jul 2023 21:50:36 +0200 Subject: [PATCH 1/4] Fix bug in 'V_StripLastDir' and 'CUtlString::DirName' In some code paths of 'V_StripLastDir', the string length value was never set. The return type of this function has changed to return always return the length of the new string. An additional bug in 'CUtlString::DirName' has been fixed, where the length was only set if the trailing slashed weren't stripped. --- r5dev/public/tier1/strtools.h | 2 +- r5dev/tier1/strtools.cpp | 56 +++++++++++++++++++++-------------- r5dev/tier1/utlstring.cpp | 9 +++--- 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/r5dev/public/tier1/strtools.h b/r5dev/public/tier1/strtools.h index 9d7604c3..e1b09264 100644 --- a/r5dev/public/tier1/strtools.h +++ b/r5dev/public/tier1/strtools.h @@ -124,7 +124,7 @@ inline void V_MakeAbsolutePath(char* pOut, size_t outLen, const char* pPath, con } // Remove the final directory from the path -bool V_StripLastDir(char* dirName, size_t maxLen, size_t* newLen); +size_t V_StripLastDir(char* dirName, size_t maxLen); // Returns a pointer to the unqualified file name (no path) of a file name const char* V_UnqualifiedFileName(const char* in); // Given a path and a filename, composes "path\filename", inserting the (OS correct) separator if necessary diff --git a/r5dev/tier1/strtools.cpp b/r5dev/tier1/strtools.cpp index ecb38482..ff6160d2 100644 --- a/r5dev/tier1/strtools.cpp +++ b/r5dev/tier1/strtools.cpp @@ -772,19 +772,22 @@ V_MakeAbsolutePath(char* pOut, size_t outLen, const char* pPath, const char* pSt // Input : *dirName - // maxLen - // *newLen - -// Output : Returns true on success, false on failure. +// Output : Returns the new length of the string //----------------------------------------------------------------------------- -bool V_StripLastDir(char* dirName, size_t maxLen, size_t* newLen) +size_t V_StripLastDir(char* dirName, size_t maxLen) { - if (dirName[0] == 0 || - !V_stricmp(dirName, "./") || - !V_stricmp(dirName, ".\\")) - return false; + Assert(dirName); + + if (dirName[0] == '\0') + return 0; size_t len = V_strlen(dirName); - Assert(len < maxLen); + if (!V_stricmp(dirName, "./") || + !V_stricmp(dirName, ".\\")) + return len; + // skip trailing slash if (PATHSEPARATOR(dirName[len - 1])) { @@ -796,9 +799,10 @@ bool V_StripLastDir(char* dirName, size_t maxLen, size_t* newLen) { if (PATHSEPARATOR(dirName[len - 1])) { - dirName[len] = 0; + dirName[len] = '\0'; V_FixSlashes(dirName, CORRECT_PATH_SEPARATOR); - return true; + + return len; } else if (dirName[len - 1] == ':') { @@ -809,27 +813,33 @@ bool V_StripLastDir(char* dirName, size_t maxLen, size_t* newLen) } // If we hit a drive letter, then we're done. - // Ex: If they passed in c:\, then V_StripLastDir should return "" and false. + // Ex: If they passed in c:\, then V_StripLastDir should + // turn the string into "" and return 0. if (bHitColon) { - dirName[0] = 0; - return false; + dirName[0] = '\0'; + return 0; } - // Allow it to return an empty string and true. This can happen if something like "tf2/" is passed in. - // The correct behavior is to strip off the last directory ("tf2") and return true. - if (len == 0 && !bHitColon) + // Allow it to return an empty string and 0. This can happen if something like "tf2/" is passed in. + // The correct behavior is to strip off the last directory ("tf2") and return the new length. + if (len == 0) { - V_snprintf(dirName, maxLen, ".%c", CORRECT_PATH_SEPARATOR); - return true; + int ret = V_snprintf(dirName, maxLen, ".%c", CORRECT_PATH_SEPARATOR); + + // snprintf failed, turn the string into "" and return 0. + if (ret < 0) + { + Assert(0); + + dirName[0] = '\0'; + return 0; + } + + return ret; } - if (newLen) - { - *newLen = len; - } - - return true; + return len; } //----------------------------------------------------------------------------- diff --git a/r5dev/tier1/utlstring.cpp b/r5dev/tier1/utlstring.cpp index adf4bda5..2a6cf36c 100644 --- a/r5dev/tier1/utlstring.cpp +++ b/r5dev/tier1/utlstring.cpp @@ -430,7 +430,8 @@ void CUtlString::StripTrailingSlash() int64 nLastChar = Length() - 1; char c = m_Storage[ nLastChar ]; - if ( c == '\\' || c == '/' ) + + if ( PATHSEPARATOR( c ) ) { m_Storage[ nLastChar ] = 0; m_Storage.SetLength( m_Storage.Length() - 1 ); @@ -607,14 +608,12 @@ CUtlString CUtlString::UnqualifiedFilename() const CUtlString CUtlString::DirName( bool bStripTrailingSlash ) const { CUtlString ret( this->String() ); - size_t len = 0; + size_t len = V_StripLastDir( (char*)ret.m_Storage.Get(), ret.m_Storage.Length() ); - V_StripLastDir( (char*)ret.m_Storage.Get(), ret.m_Storage.Length(), &len ); + ret.SetLength( len ); if (bStripTrailingSlash) ret.StripTrailingSlash(); - else - ret.SetLength(len); // StripTrailingSlash sets this, but if we skip it then nothing sets it. return ret; } From 77299dd3de29e4ef3207215a94bbd962d9954287 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Wed, 5 Jul 2023 22:08:32 +0200 Subject: [PATCH 2/4] Remove extraneous 'V_FixSlashes' call No need to fix slashes when stripping last dir. We can always fix them up on a subsequent call to 'V_FixSlashes' if we ever need to. --- r5dev/tier1/strtools.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/r5dev/tier1/strtools.cpp b/r5dev/tier1/strtools.cpp index ff6160d2..6c0cbfd0 100644 --- a/r5dev/tier1/strtools.cpp +++ b/r5dev/tier1/strtools.cpp @@ -800,8 +800,6 @@ size_t V_StripLastDir(char* dirName, size_t maxLen) if (PATHSEPARATOR(dirName[len - 1])) { dirName[len] = '\0'; - V_FixSlashes(dirName, CORRECT_PATH_SEPARATOR); - return len; } else if (dirName[len - 1] == ':') From 5939e05331a123087158bb96d4b554a11d2768c5 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Wed, 5 Jul 2023 23:04:07 +0200 Subject: [PATCH 3/4] Fix incorrect VPK tree during build This commit fixes the following bugs: - Use of STL map on CUtlString, CUtlString does not have a lesser operator, thus it always fails to check whether or not the string is already present in the map. The tree structure cannot contain duplicates per tree scope. - Use of backward slashes in vpk file tree structure, path separators should always be a forwards slash. The code was originally designed to be forwards slash only, but some paths were adjusted to take backwards slash, as the KeyValues class treats the '/' char as a delimiter for subfields. This has therefore lead into the whole code changing to it. The code has now been changed to the point where only the KeyValues utilities take the backward slash paths. - Trailing slash character on file path, that ended up getting written as-is in the path tree. The VPK tree builder is now part of a separate nested class in 'VPKDir_t'. --- r5dev/vpklib/packedstore.cpp | 150 ++++++++++++++++++++++------------- r5dev/vpklib/packedstore.h | 17 +++- 2 files changed, 108 insertions(+), 59 deletions(-) diff --git a/r5dev/vpklib/packedstore.cpp b/r5dev/vpklib/packedstore.cpp index d468caa0..6eedd014 100644 --- a/r5dev/vpklib/packedstore.cpp +++ b/r5dev/vpklib/packedstore.cpp @@ -142,7 +142,9 @@ bool CPackedStore::GetEntryValues(CUtlVector& entryValues, } CUtlString fileName; + fileName.Format("%s%s", workspacePath.Get(), pszFileName); + fileName.FixSlashes('/'); if (ShouldPrune(fileName, ignoreList)) { @@ -203,9 +205,7 @@ CUtlString CPackedStore::GetLevelName(const CUtlString& dirFileName) const KeyValues* CPackedStore::GetManifest(const CUtlString& workspacePath, const CUtlString& manifestFile) const { CUtlString outPath; - outPath.Format("%s%s%s.txt", workspacePath.Get(), "manifest/", manifestFile.Get()); - outPath.FixSlashes(); KeyValues* pManifestKV = FileSystem()->LoadKeyValues(IFileSystem::TYPE_COMMON, outPath.Get(), "PLATFORM"); return pManifestKV; @@ -220,9 +220,7 @@ KeyValues* CPackedStore::GetManifest(const CUtlString& workspacePath, const CUtl bool CPackedStore::GetIgnoreList(CUtlVector& ignoreList, const CUtlString& workspacePath) const { CUtlString toIgnore; - toIgnore.Format("%s%s", workspacePath.Get(), VPK_IGNORE_FILE); - toIgnore.FixSlashes(); FileHandle_t hIgnoreFile = FileSystem()->Open(toIgnore.Get(), "rt", "PLATFORM"); if (!hIgnoreFile) @@ -276,7 +274,7 @@ CUtlString CPackedStore::FormatEntryPath(const CUtlString& filePath, result.Format("%s%s.%s", isRoot ? "" : pszFilePath, fileName.Get(), fileExt.Get()); - result.FixSlashes(); + result.FixSlashes('/'); return result; } @@ -296,7 +294,13 @@ void CPackedStore::BuildManifest(const CUtlVector& entryBlocks, const VPKEntryBlock_t& entry = entryBlocks[i]; const VPKChunkDescriptor_t& descriptor = entry.m_Fragments[0]; - KeyValues* pEntryKV = pManifestKV->FindKey(entry.m_EntryPath.Get(), true); + // Copy, because we need to change the '/' slashes into + // '\\'. KeyValues has the '/' character reserved for + // delimiting subfields. + CUtlString entryPath = entry.m_EntryPath; + entryPath.FixSlashes('\\'); + + KeyValues* pEntryKV = pManifestKV->FindKey(entryPath.Get(), true); pEntryKV->SetInt("preloadSize", entry.m_iPreloadSize); pEntryKV->SetInt("loadFlags", descriptor.m_nLoadFlags); @@ -307,7 +311,6 @@ void CPackedStore::BuildManifest(const CUtlVector& entryBlocks, CUtlString outPath; outPath.Format("%s%s%s.txt", workspacePath.Get(), "manifest/", manifestName.Get()); - outPath.FixSlashes(); CUtlBuffer outBuf(int64_t(0), 0, CUtlBuffer::TEXT_BUFFER); kv.RecursiveSaveToFile(outBuf, 0); @@ -426,7 +429,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vpkPair, const char* workspace { CUtlString workspacePath(workspaceName); workspacePath.AppendSlash(); - workspacePath.FixSlashes(); + workspacePath.FixSlashes('/'); CUtlString packFilePath; CUtlString dirFilePath; @@ -486,7 +489,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vpkPair, const char* workspace FileSystem()->Seek(hAsset, 0, FileSystemSeek_t::FILESYSTEM_SEEK_HEAD); DevMsg(eDLL_T::FS, "Packing entry '%i' ('%s')\n", i, szDestPath); - entryBlocks.AddToTail(VPKEntryBlock_t( + int index = entryBlocks.AddToTail(VPKEntryBlock_t( pBuf.get(), nLen, FileSystem()->Tell(hPackFile), @@ -496,7 +499,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vpkPair, const char* workspace entryValue.m_nTextureFlags, CUtlString(szDestPath))); - VPKEntryBlock_t& entryBlock = entryBlocks[i]; + VPKEntryBlock_t& entryBlock = entryBlocks[index]; FOR_EACH_VEC(entryBlock.m_Fragments, j) { @@ -522,7 +525,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vpkPair, const char* workspace if (lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS) { Warning(eDLL_T::FS, "Status '%d' for chunk '%i' within entry '%i' in block '%hu' (chunk packed without compression)\n", - lzCompStatus, j, i, entryBlocks[i].m_iPackFileIndex); + lzCompStatus, j, i, entryBlock.m_iPackFileIndex); descriptor.m_nCompressedSize = descriptor.m_nUncompressedSize; } @@ -555,8 +558,9 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vpkPair, const char* workspace void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspaceName) { CUtlString workspacePath(workspaceName); + workspacePath.AppendSlash(); - workspacePath.FixSlashes(); + workspacePath.FixSlashes('/'); if (vpkDir.m_Header.m_nHeaderMarker != VPK_HEADER_MARKER || vpkDir.m_Header.m_nMajorVersion != VPK_MAJOR_VERSION || @@ -683,8 +687,8 @@ VPKEntryBlock_t::VPKEntryBlock_t(FileHandle_t hDirFile, const char* pEntryPath) { m_EntryPath = pEntryPath; // Set the entry path. - m_EntryPath.FixSlashes(); // Fix slashes and remove space character representing VPK root. - m_EntryPath = m_EntryPath.Replace(" " CORRECT_PATH_SEPARATOR_S, ""); + m_EntryPath.FixSlashes('/'); // Fix slashes and remove space character representing VPK root. + m_EntryPath = m_EntryPath.Replace(" /", ""); FileSystem()->Read(&m_nFileCRC, sizeof(uint32_t), hDirFile); // FileSystem()->Read(&m_iPreloadSize, sizeof(uint16_t), hDirFile); // @@ -720,6 +724,8 @@ VPKEntryBlock_t::VPKEntryBlock_t(const uint8_t* pData, size_t nLen, int64_t nOff m_iPackFileIndex = iPackFileIndex; m_EntryPath = pEntryPath; + m_EntryPath.FixSlashes('/'); + size_t nFragmentCount = (nLen + ENTRY_MAX_LEN - 1) / ENTRY_MAX_LEN; size_t nFileSize = nLen; int64_t nCurrentOffset = nOffset; @@ -981,26 +987,89 @@ void VPKDir_t::WriteTreeSize(FileHandle_t hDirectoryFile) const } //----------------------------------------------------------------------------- -// Purpose: writes the vpk chunk descriptors +// Purpose: builds the vpk directory tree +// Input : &entryBlocks - +//----------------------------------------------------------------------------- +void VPKDir_t::CTreeBuilder::BuildTree(const CUtlVector& entryBlocks) +{ + FOR_EACH_VEC(entryBlocks, i) + { + const VPKEntryBlock_t& entryBlock = entryBlocks[i]; + + CUtlString fileExt = entryBlock.m_EntryPath.GetExtension(); + CUtlString filePath = entryBlock.m_EntryPath.DirName(); + + if (!filePath.IsEmpty() && filePath[0] == '.') + { + // Has to be padded with a space character if empty [root]. + filePath = " "; + } + + /********************************************************************** + * The code below creates a directory tree structure as follows: + * + * Extension0 + * | + * |--- Path0 + * | | + * | File0 + * | File1 + * | File2 + * | + * |--- Path1 + * | + * File0 + * File1 + * File2 + * ... + * + * A tree scope cannot contain duplicate elements, + * which ultimately means that: + * + * - An extension is only written once to the tree. + * - A file path is only written once per extension tree. + * - A file name is only written once per file path tree. + **********************************************************************/ + const char* pFileExt = fileExt.Get(); + auto extIt = m_FileTree.find(pFileExt); + + if (extIt == m_FileTree.end()) + { + extIt = m_FileTree.insert({ pFileExt, PathContainer_t() }).first; + } + + PathContainer_t& pathTree = extIt->second; + + const char* pFilePath = filePath.Get(); + auto pathIt = pathTree.find(pFilePath); + + if (pathIt == pathTree.end()) + { + pathIt = pathTree.insert({ pFilePath, std::list() }).first; + } + + pathIt->second.push_back(entryBlock); + } +} + +//----------------------------------------------------------------------------- +// Purpose: writes the vpk directory tree // Input : hDirectoryFile - -// &vMap - // Output : number of descriptors written //----------------------------------------------------------------------------- -uint64_t VPKDir_t::WriteDescriptor(FileHandle_t hDirectoryFile, - std::map>>& vMap) const +uint64_t VPKDir_t::CTreeBuilder::WriteTree(FileHandle_t hDirectoryFile) const { uint64_t nDescriptors = NULL; - for (auto& iKeyValue : vMap) + for (auto& iKeyValue : m_FileTree) { - FileSystem()->Write(iKeyValue.first.Get(), int(iKeyValue.first.Length() + 1), hDirectoryFile); + FileSystem()->Write(iKeyValue.first.c_str(), int(iKeyValue.first.length() + 1), hDirectoryFile); for (auto& jKeyValue : iKeyValue.second) { - FileSystem()->Write(jKeyValue.first.Get(), int(jKeyValue.first.Length() + 1), hDirectoryFile); + FileSystem()->Write(jKeyValue.first.c_str(), int(jKeyValue.first.length() + 1), hDirectoryFile); for (auto& vEntry : jKeyValue.second) { CUtlString entryPath = vEntry.m_EntryPath.UnqualifiedFilename().StripExtension(); - FileSystem()->Write(entryPath.Get(), int(entryPath.Length() + 1), hDirectoryFile); FileSystem()->Write(&vEntry.m_nFileCRC, sizeof(uint32_t), hDirectoryFile); @@ -1038,37 +1107,6 @@ uint64_t VPKDir_t::WriteDescriptor(FileHandle_t hDirectoryFile, return nDescriptors; } -//----------------------------------------------------------------------------- -// Purpose: builds the vpk directory tree -// Input : &vEntryBlocks - -// &vMap - -//----------------------------------------------------------------------------- -void VPKDir_t::BuildDirectoryTree(const CUtlVector& entryBlocks, - std::map>>& vMap) const -{ - FOR_EACH_VEC(entryBlocks, i) - { - const VPKEntryBlock_t& entryBlock = entryBlocks[i]; - - CUtlString fileExt = entryBlock.m_EntryPath.GetExtension(); - CUtlString filePath = entryBlock.m_EntryPath.DirName(); - - if (!filePath.IsEmpty() && filePath[0] == '.') - { - filePath = " "; // Has to be padded with a space character if empty [root]. - } - if (!vMap.count(fileExt)) - { - vMap.insert({ fileExt, std::map>() }); - } - if (!vMap[fileExt].count(filePath)) - { - vMap[fileExt].insert({ filePath, std::list() }); - } - vMap[fileExt][filePath].push_back(entryBlock); - } -} - //----------------------------------------------------------------------------- // Purpose: builds the vpk directory file // Input : &svDirectoryPath - @@ -1083,11 +1121,11 @@ void VPKDir_t::BuildDirectoryFile(const CUtlString& directoryPath, const CUtlVec return; } - auto vMap = std::map>>(); - BuildDirectoryTree(entryBlocks, vMap); + CTreeBuilder treeBuilder; + treeBuilder.BuildTree(entryBlocks); WriteHeader(hDirectoryFile); - uint64_t nDescriptors = WriteDescriptor(hDirectoryFile, vMap); + uint64_t nDescriptors = treeBuilder.WriteTree(hDirectoryFile); m_Header.m_nDirectorySize = static_cast(FileSystem()->Tell(hDirectoryFile) - sizeof(VPKDirHeader_t)); WriteTreeSize(hDirectoryFile); diff --git a/r5dev/vpklib/packedstore.h b/r5dev/vpklib/packedstore.h index 3b7943fc..f3c34dcb 100644 --- a/r5dev/vpklib/packedstore.h +++ b/r5dev/vpklib/packedstore.h @@ -128,9 +128,22 @@ struct VPKDir_t CUtlVector m_PackFiles; // Vector of pack file names. CUtlString m_DirFilePath; // Path to vpk_dir file. + class CTreeBuilder + { + public: + typedef std::map> PathContainer_t; + typedef std::map TypeContainer_t; + + void BuildTree(const CUtlVector& entryBlocks); + uint64_t WriteTree(FileHandle_t hDirectoryFile) const; + + private: + TypeContainer_t m_FileTree; + }; + VPKDir_t() { - m_Header.m_nHeaderMarker = VPK_HEADER_MARKER; m_Header.m_nMajorVersion = VPK_MAJOR_VERSION; + m_Header.m_nHeaderMarker = VPK_HEADER_MARKER; m_Header.m_nMajorVersion = VPK_MAJOR_VERSION; m_Header.m_nMinorVersion = VPK_MINOR_VERSION; m_Header.m_nDirectorySize = NULL, m_Header.m_nSignatureSize = NULL; m_PackFileCount = NULL; }; @@ -144,9 +157,7 @@ struct VPKDir_t void WriteHeader(FileHandle_t hDirectoryFile) const; void WriteTreeSize(FileHandle_t hDirectoryFile) const; - uint64_t WriteDescriptor(FileHandle_t hDirectoryFile, std::map>>& vMap) const; - void BuildDirectoryTree(const CUtlVector& entryBlocks, std::map>>& vMap) const; void BuildDirectoryFile(const CUtlString& svDirectoryFile, const CUtlVector& entryBlocks); }; From c215fcc171bb74090a8e29f690b725e28295bd04 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Thu, 6 Jul 2023 00:49:17 +0200 Subject: [PATCH 4/4] VPK utility code improvements - Only process referenced pack files, previously, if an entry in the VPK file was build into a pack of index 128, the code would generate a list of pack files from 0 to 128. - Added documentation to VPK structure. --- r5dev/public/ipackedstore.h | 4 +- r5dev/vpklib/packedstore.cpp | 36 ++++------- r5dev/vpklib/packedstore.h | 122 ++++++++++++++++++++++++----------- 3 files changed, 99 insertions(+), 63 deletions(-) diff --git a/r5dev/public/ipackedstore.h b/r5dev/public/ipackedstore.h index 76a5118b..4e26ea44 100644 --- a/r5dev/public/ipackedstore.h +++ b/r5dev/public/ipackedstore.h @@ -1,7 +1,7 @@ #ifndef IPACKEDSTORE_H #define IPACKEDSTORE_H -enum class EPackedLoadFlags : int +enum EPackedLoadFlags { LOAD_NONE, LOAD_VISIBLE = 1 << 0, // Visible to FileSystem. @@ -11,7 +11,7 @@ enum class EPackedLoadFlags : int LOAD_TEXTURE_UNK2 = 1 << 20, }; -enum class EPackedTextureFlags : short +enum EPackedTextureFlags { TEXTURE_NONE, TEXTURE_DEFAULT = 1 << 3, diff --git a/r5dev/vpklib/packedstore.cpp b/r5dev/vpklib/packedstore.cpp index 6eedd014..9ac89222 100644 --- a/r5dev/vpklib/packedstore.cpp +++ b/r5dev/vpklib/packedstore.cpp @@ -582,9 +582,9 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace BuildManifest(vpkDir.m_EntryBlocks, workspacePath, GetLevelName(vpkDir.m_DirFilePath)); const CUtlString basePath = vpkDir.m_DirFilePath.StripFilename(false); - FOR_EACH_VEC(vpkDir.m_PackFiles, i) + for (uint16_t packFileIndex : vpkDir.m_PakFileIndices) { - const CUtlString packFile = basePath + vpkDir.m_PackFiles[i]; + const CUtlString packFile = basePath + vpkDir.GetPackFileNameForIndex(packFileIndex); // Read from each pack file. FileHandle_t hPackFile = FileSystem()->Open(packFile.Get(), "rb", "GAME"); @@ -598,7 +598,7 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace { const VPKEntryBlock_t& entryBlock = vpkDir.m_EntryBlocks[j]; - if (entryBlock.m_iPackFileIndex != uint16_t(i)) + if (entryBlock.m_iPackFileIndex != packFileIndex) { // Chunk doesn't belongs to this block. continue; @@ -616,7 +616,8 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace continue; } - DevMsg(eDLL_T::FS, "Unpacking entry '%i' from block '%i' ('%s')\n", j, i, entryBlock.m_EntryPath.Get()); + DevMsg(eDLL_T::FS, "Unpacking entry '%i' from block '%i' ('%s')\n", + j, entryBlock.m_iPackFileIndex, entryBlock.m_EntryPath.Get()); FOR_EACH_VEC(entryBlock.m_Fragments, k) { @@ -642,8 +643,8 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS) { - Error(eDLL_T::FS, NO_ERROR, "Status '%d' for chunk '%zu' within entry '%zu' in block '%hu' (chunk not decompressed)\n", - lzDecompStatus, k, j, i); + Error(eDLL_T::FS, NO_ERROR, "Status '%d' for chunk '%i' within entry '%i' in block '%hu' (chunk not decompressed)\n", + lzDecompStatus, k, j, packFileIndex); } else // If successfully decompressed, write to file. { @@ -906,37 +907,26 @@ void VPKDir_t::Init(const CUtlString& dirFilePath) FileSystem()->Read(&m_Header.m_nSignatureSize, sizeof(uint32_t), hDirFile); // g_pPackedStore->GetEntryBlocks(m_EntryBlocks, hDirFile); - m_DirFilePath = dirFilePath; // Set path to vpk directory file. - m_PackFileCount = 0; + // Obtain every referenced pack file from the directory tree. FOR_EACH_VEC(m_EntryBlocks, i) { const VPKEntryBlock_t& entryBlock = m_EntryBlocks[i]; - - if (entryBlock.m_iPackFileIndex > m_PackFileCount) - { - m_PackFileCount = entryBlock.m_iPackFileIndex; - } - } - - for (uint16_t i = 0; i < m_PackFileCount + 1; i++) - { - m_PackFiles.AddToTail(GetPackFile(dirFilePath, i)); + m_PakFileIndices.insert(entryBlock.m_iPackFileIndex); } FileSystem()->Close(hDirFile); } //----------------------------------------------------------------------------- -// Purpose: formats pack file path for specific directory file -// Input : &directoryPath - -// iPackFileIndex - +// Purpose: formats pack file path for specified patch +// Input : iPackFileIndex - (patch) // output : string //----------------------------------------------------------------------------- -CUtlString VPKDir_t::GetPackFile(const CUtlString& directoryPath, uint16_t iPackFileIndex) const +CUtlString VPKDir_t::GetPackFileNameForIndex(uint16_t iPackFileIndex) const { - CUtlString packChunkName = StripLocalePrefix(directoryPath); + CUtlString packChunkName = StripLocalePrefix(m_DirFilePath); CUtlString packChunkIndex; packChunkIndex.Format("pak000_%03d", iPackFileIndex); diff --git a/r5dev/vpklib/packedstore.h b/r5dev/vpklib/packedstore.h index f3c34dcb..b3ff0bbd 100644 --- a/r5dev/vpklib/packedstore.h +++ b/r5dev/vpklib/packedstore.h @@ -44,10 +44,18 @@ static const char* const DIR_LOCALE[] "tchinese" }; +//----------------------------------------------------------------------------- +// KeyValues structure for the VPK manifest file. This struct gets populated by +// the VPK's corresponding manifest file, which ultimately determines how each +// asset is getting packed into the VPK. +//----------------------------------------------------------------------------- struct VPKKeyValues_t { - static constexpr uint16_t TEXTURE_FLAGS_DEFAULT = static_cast(EPackedTextureFlags::TEXTURE_DEFAULT); - static constexpr uint32_t LOAD_FLAGS_DEFAULT = static_cast(EPackedLoadFlags::LOAD_VISIBLE) | static_cast(EPackedLoadFlags::LOAD_CACHE); + static constexpr uint16_t TEXTURE_FLAGS_DEFAULT = + EPackedTextureFlags::TEXTURE_DEFAULT; + + static constexpr uint32_t LOAD_FLAGS_DEFAULT = + EPackedLoadFlags::LOAD_VISIBLE | EPackedLoadFlags::LOAD_CACHE; CUtlString m_EntryPath; uint16_t m_iPreloadSize; @@ -56,17 +64,29 @@ struct VPKKeyValues_t bool m_bUseCompression; bool m_bDeduplicate; - VPKKeyValues_t(const CUtlString& svEntryPath = "", uint16_t iPreloadSize = NULL, uint32_t nLoadFlags = LOAD_FLAGS_DEFAULT, - uint16_t nTextureFlags = TEXTURE_FLAGS_DEFAULT, bool bUseCompression = true, bool bDeduplicate = true); + VPKKeyValues_t(const CUtlString& svEntryPath = "", + uint16_t iPreloadSize = NULL, + uint32_t nLoadFlags = LOAD_FLAGS_DEFAULT, + uint16_t nTextureFlags = TEXTURE_FLAGS_DEFAULT, + bool bUseCompression = true, bool bDeduplicate = true); }; +//----------------------------------------------------------------------------- +// An asset packed into a VPK is carved into 'ENTRY_MAX_LEN' chunks, the chunk +// is then optionally compressed. A chunk is NOT compressed if the compressed +// size equals the uncompressed size. +//----------------------------------------------------------------------------- struct VPKChunkDescriptor_t { - uint32_t m_nLoadFlags; // Load flags. - uint16_t m_nTextureFlags; // Texture flags (only used if the entry is a vtf). - uint64_t m_nPackFileOffset; // Offset in pack file. - uint64_t m_nCompressedSize; // Compressed size of chunk. - uint64_t m_nUncompressedSize; // Uncompressed size of chunk. + uint32_t m_nLoadFlags; + + // Texture flags (only used if the entry is a vtf). + uint16_t m_nTextureFlags; + + // Offset in pack file. + uint64_t m_nPackFileOffset; + uint64_t m_nCompressedSize; + uint64_t m_nUncompressedSize; VPKChunkDescriptor_t() : m_nLoadFlags(0) @@ -77,16 +97,27 @@ struct VPKChunkDescriptor_t { } VPKChunkDescriptor_t(FileHandle_t hDirectoryFile); - VPKChunkDescriptor_t(uint32_t nLoadFlags, uint16_t nTextureFlags, uint64_t nPackFileOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize); + VPKChunkDescriptor_t(uint32_t nLoadFlags, uint16_t nTextureFlags, + uint64_t nPackFileOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize); }; +//----------------------------------------------------------------------------- +// An asset packed into a VPK is represented as an entry block. +//----------------------------------------------------------------------------- struct VPKEntryBlock_t { - uint32_t m_nFileCRC; // Crc32 for the uncompressed entry. - uint16_t m_iPreloadSize; // Preload bytes. - uint16_t m_iPackFileIndex; // Index of the pack file that contains this entry. - CUtlVector m_Fragments; // Vector of all the chunks of a given entry (chunks have a size limit of 1 MiB, anything over this limit is fragmented into smaller chunks). - CUtlString m_EntryPath; // Path to entry within vpk. + // Crc32 for the uncompressed entry. + uint32_t m_nFileCRC; + uint16_t m_iPreloadSize; + + // Index of the pack file that contains this entry. + uint16_t m_iPackFileIndex; + + // Vector of all the chunks of a given entry + // (chunks have a size limit of 1 MiB, anything + // over this limit is fragmented into smaller chunks). + CUtlVector m_Fragments; + CUtlString m_EntryPath; VPKEntryBlock_t(FileHandle_t pFile, const char* svEntryPath); VPKEntryBlock_t(const uint8_t* pData, size_t nLen, int64_t nOffset, uint16_t iPreloadSize, @@ -103,30 +134,30 @@ struct VPKEntryBlock_t } }; +//----------------------------------------------------------------------------- +// The VPK directory file header. +//----------------------------------------------------------------------------- struct VPKDirHeader_t { - uint32_t m_nHeaderMarker; // File magic. - uint16_t m_nMajorVersion; // Vpk major version. - uint16_t m_nMinorVersion; // Vpk minor version. - uint32_t m_nDirectorySize; // Directory tree size. - uint32_t m_nSignatureSize; // Directory signature. -}; - -struct VPKPair_t -{ - CUtlString m_PackName; - CUtlString m_DirName; - - VPKPair_t(const char* svLocale, const char* svTarget, const char* svLevel, int nPatch); + uint32_t m_nHeaderMarker; // File magic. + uint16_t m_nMajorVersion; // Vpk major version. + uint16_t m_nMinorVersion; // Vpk minor version. + uint32_t m_nDirectorySize; // Directory tree size. + uint32_t m_nSignatureSize; // Directory signature. }; +//----------------------------------------------------------------------------- +// The VPK directory tree structure. +//----------------------------------------------------------------------------- struct VPKDir_t { - VPKDirHeader_t m_Header; // Dir header. - CUtlVector m_EntryBlocks; // Vector of entry blocks. - uint16_t m_PackFileCount; // Number of pack patches (pack file count-1). - CUtlVector m_PackFiles; // Vector of pack file names. - CUtlString m_DirFilePath; // Path to vpk_dir file. + VPKDirHeader_t m_Header; + CUtlVector m_EntryBlocks; + CUtlString m_DirFilePath; + + // This set only contains packfile indices used + // by the directory tree, notated as pak000_xxx. + std::set m_PakFileIndices; class CTreeBuilder { @@ -145,7 +176,6 @@ struct VPKDir_t { m_Header.m_nHeaderMarker = VPK_HEADER_MARKER; m_Header.m_nMajorVersion = VPK_MAJOR_VERSION; m_Header.m_nMinorVersion = VPK_MINOR_VERSION; m_Header.m_nDirectorySize = NULL, m_Header.m_nSignatureSize = NULL; - m_PackFileCount = NULL; }; VPKDir_t(const CUtlString& svDirectoryFile); VPKDir_t(const CUtlString& svDirectoryFile, bool bSanitizeName); @@ -153,7 +183,7 @@ struct VPKDir_t void Init(const CUtlString& svPath); CUtlString StripLocalePrefix(const CUtlString& svDirectoryFile) const; - CUtlString GetPackFile(const CUtlString& svDirectoryPath, uint16_t iPackFileIndex) const; + CUtlString GetPackFileNameForIndex(uint16_t iPackFileIndex) const; void WriteHeader(FileHandle_t hDirectoryFile) const; void WriteTreeSize(FileHandle_t hDirectoryFile) const; @@ -161,6 +191,22 @@ struct VPKDir_t void BuildDirectoryFile(const CUtlString& svDirectoryFile, const CUtlVector& entryBlocks); }; +//----------------------------------------------------------------------------- +// Contains the VPK directory name, and the pack file name. Used for building +// the VPK file. +// !TODO[ AMOS ]: Remove this when patching is implemented! +//----------------------------------------------------------------------------- +struct VPKPair_t +{ + CUtlString m_PackName; + CUtlString m_DirName; + + VPKPair_t(const char* svLocale, const char* svTarget, const char* svLevel, int nPatch); +}; + +//----------------------------------------------------------------------------- +// VPK utility class. +//----------------------------------------------------------------------------- class CPackedStore { public: @@ -190,11 +236,11 @@ public: void UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspaceName = ""); private: - lzham_compress_params m_lzCompParams; // LZham compression parameters. - lzham_decompress_params m_lzDecompParams; // LZham decompression parameters. + lzham_compress_params m_lzCompParams; // LZham compression parameters. + lzham_decompress_params m_lzDecompParams; // LZham decompression parameters. std::unordered_map m_ChunkHashMap; }; /////////////////////////////////////////////////////////////////////////////// extern CPackedStore* g_pPackedStore; -#endif // PACKEDSTORE_H \ No newline at end of file +#endif // PACKEDSTORE_H