From 5939e05331a123087158bb96d4b554a11d2768c5 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Wed, 5 Jul 2023 23:04:07 +0200 Subject: [PATCH] Fix incorrect VPK tree during build This commit fixes the following bugs: - Use of STL map on CUtlString, CUtlString does not have a lesser operator, thus it always fails to check whether or not the string is already present in the map. The tree structure cannot contain duplicates per tree scope. - Use of backward slashes in vpk file tree structure, path separators should always be a forwards slash. The code was originally designed to be forwards slash only, but some paths were adjusted to take backwards slash, as the KeyValues class treats the '/' char as a delimiter for subfields. This has therefore lead into the whole code changing to it. The code has now been changed to the point where only the KeyValues utilities take the backward slash paths. - Trailing slash character on file path, that ended up getting written as-is in the path tree. The VPK tree builder is now part of a separate nested class in 'VPKDir_t'. --- r5dev/vpklib/packedstore.cpp | 150 ++++++++++++++++++++++------------- r5dev/vpklib/packedstore.h | 17 +++- 2 files changed, 108 insertions(+), 59 deletions(-) diff --git a/r5dev/vpklib/packedstore.cpp b/r5dev/vpklib/packedstore.cpp index d468caa0..6eedd014 100644 --- a/r5dev/vpklib/packedstore.cpp +++ b/r5dev/vpklib/packedstore.cpp @@ -142,7 +142,9 @@ bool CPackedStore::GetEntryValues(CUtlVector& entryValues, } CUtlString fileName; + fileName.Format("%s%s", workspacePath.Get(), pszFileName); + fileName.FixSlashes('/'); if (ShouldPrune(fileName, ignoreList)) { @@ -203,9 +205,7 @@ CUtlString CPackedStore::GetLevelName(const CUtlString& dirFileName) const KeyValues* CPackedStore::GetManifest(const CUtlString& workspacePath, const CUtlString& manifestFile) const { CUtlString outPath; - outPath.Format("%s%s%s.txt", workspacePath.Get(), "manifest/", manifestFile.Get()); - outPath.FixSlashes(); KeyValues* pManifestKV = FileSystem()->LoadKeyValues(IFileSystem::TYPE_COMMON, outPath.Get(), "PLATFORM"); return pManifestKV; @@ -220,9 +220,7 @@ KeyValues* CPackedStore::GetManifest(const CUtlString& workspacePath, const CUtl bool CPackedStore::GetIgnoreList(CUtlVector& ignoreList, const CUtlString& workspacePath) const { CUtlString toIgnore; - toIgnore.Format("%s%s", workspacePath.Get(), VPK_IGNORE_FILE); - toIgnore.FixSlashes(); FileHandle_t hIgnoreFile = FileSystem()->Open(toIgnore.Get(), "rt", "PLATFORM"); if (!hIgnoreFile) @@ -276,7 +274,7 @@ CUtlString CPackedStore::FormatEntryPath(const CUtlString& filePath, result.Format("%s%s.%s", isRoot ? "" : pszFilePath, fileName.Get(), fileExt.Get()); - result.FixSlashes(); + result.FixSlashes('/'); return result; } @@ -296,7 +294,13 @@ void CPackedStore::BuildManifest(const CUtlVector& entryBlocks, const VPKEntryBlock_t& entry = entryBlocks[i]; const VPKChunkDescriptor_t& descriptor = entry.m_Fragments[0]; - KeyValues* pEntryKV = pManifestKV->FindKey(entry.m_EntryPath.Get(), true); + // Copy, because we need to change the '/' slashes into + // '\\'. KeyValues has the '/' character reserved for + // delimiting subfields. + CUtlString entryPath = entry.m_EntryPath; + entryPath.FixSlashes('\\'); + + KeyValues* pEntryKV = pManifestKV->FindKey(entryPath.Get(), true); pEntryKV->SetInt("preloadSize", entry.m_iPreloadSize); pEntryKV->SetInt("loadFlags", descriptor.m_nLoadFlags); @@ -307,7 +311,6 @@ void CPackedStore::BuildManifest(const CUtlVector& entryBlocks, CUtlString outPath; outPath.Format("%s%s%s.txt", workspacePath.Get(), "manifest/", manifestName.Get()); - outPath.FixSlashes(); CUtlBuffer outBuf(int64_t(0), 0, CUtlBuffer::TEXT_BUFFER); kv.RecursiveSaveToFile(outBuf, 0); @@ -426,7 +429,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vpkPair, const char* workspace { CUtlString workspacePath(workspaceName); workspacePath.AppendSlash(); - workspacePath.FixSlashes(); + workspacePath.FixSlashes('/'); CUtlString packFilePath; CUtlString dirFilePath; @@ -486,7 +489,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vpkPair, const char* workspace FileSystem()->Seek(hAsset, 0, FileSystemSeek_t::FILESYSTEM_SEEK_HEAD); DevMsg(eDLL_T::FS, "Packing entry '%i' ('%s')\n", i, szDestPath); - entryBlocks.AddToTail(VPKEntryBlock_t( + int index = entryBlocks.AddToTail(VPKEntryBlock_t( pBuf.get(), nLen, FileSystem()->Tell(hPackFile), @@ -496,7 +499,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vpkPair, const char* workspace entryValue.m_nTextureFlags, CUtlString(szDestPath))); - VPKEntryBlock_t& entryBlock = entryBlocks[i]; + VPKEntryBlock_t& entryBlock = entryBlocks[index]; FOR_EACH_VEC(entryBlock.m_Fragments, j) { @@ -522,7 +525,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vpkPair, const char* workspace if (lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS) { Warning(eDLL_T::FS, "Status '%d' for chunk '%i' within entry '%i' in block '%hu' (chunk packed without compression)\n", - lzCompStatus, j, i, entryBlocks[i].m_iPackFileIndex); + lzCompStatus, j, i, entryBlock.m_iPackFileIndex); descriptor.m_nCompressedSize = descriptor.m_nUncompressedSize; } @@ -555,8 +558,9 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vpkPair, const char* workspace void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspaceName) { CUtlString workspacePath(workspaceName); + workspacePath.AppendSlash(); - workspacePath.FixSlashes(); + workspacePath.FixSlashes('/'); if (vpkDir.m_Header.m_nHeaderMarker != VPK_HEADER_MARKER || vpkDir.m_Header.m_nMajorVersion != VPK_MAJOR_VERSION || @@ -683,8 +687,8 @@ VPKEntryBlock_t::VPKEntryBlock_t(FileHandle_t hDirFile, const char* pEntryPath) { m_EntryPath = pEntryPath; // Set the entry path. - m_EntryPath.FixSlashes(); // Fix slashes and remove space character representing VPK root. - m_EntryPath = m_EntryPath.Replace(" " CORRECT_PATH_SEPARATOR_S, ""); + m_EntryPath.FixSlashes('/'); // Fix slashes and remove space character representing VPK root. + m_EntryPath = m_EntryPath.Replace(" /", ""); FileSystem()->Read(&m_nFileCRC, sizeof(uint32_t), hDirFile); // FileSystem()->Read(&m_iPreloadSize, sizeof(uint16_t), hDirFile); // @@ -720,6 +724,8 @@ VPKEntryBlock_t::VPKEntryBlock_t(const uint8_t* pData, size_t nLen, int64_t nOff m_iPackFileIndex = iPackFileIndex; m_EntryPath = pEntryPath; + m_EntryPath.FixSlashes('/'); + size_t nFragmentCount = (nLen + ENTRY_MAX_LEN - 1) / ENTRY_MAX_LEN; size_t nFileSize = nLen; int64_t nCurrentOffset = nOffset; @@ -981,26 +987,89 @@ void VPKDir_t::WriteTreeSize(FileHandle_t hDirectoryFile) const } //----------------------------------------------------------------------------- -// Purpose: writes the vpk chunk descriptors +// Purpose: builds the vpk directory tree +// Input : &entryBlocks - +//----------------------------------------------------------------------------- +void VPKDir_t::CTreeBuilder::BuildTree(const CUtlVector& entryBlocks) +{ + FOR_EACH_VEC(entryBlocks, i) + { + const VPKEntryBlock_t& entryBlock = entryBlocks[i]; + + CUtlString fileExt = entryBlock.m_EntryPath.GetExtension(); + CUtlString filePath = entryBlock.m_EntryPath.DirName(); + + if (!filePath.IsEmpty() && filePath[0] == '.') + { + // Has to be padded with a space character if empty [root]. + filePath = " "; + } + + /********************************************************************** + * The code below creates a directory tree structure as follows: + * + * Extension0 + * | + * |--- Path0 + * | | + * | File0 + * | File1 + * | File2 + * | + * |--- Path1 + * | + * File0 + * File1 + * File2 + * ... + * + * A tree scope cannot contain duplicate elements, + * which ultimately means that: + * + * - An extension is only written once to the tree. + * - A file path is only written once per extension tree. + * - A file name is only written once per file path tree. + **********************************************************************/ + const char* pFileExt = fileExt.Get(); + auto extIt = m_FileTree.find(pFileExt); + + if (extIt == m_FileTree.end()) + { + extIt = m_FileTree.insert({ pFileExt, PathContainer_t() }).first; + } + + PathContainer_t& pathTree = extIt->second; + + const char* pFilePath = filePath.Get(); + auto pathIt = pathTree.find(pFilePath); + + if (pathIt == pathTree.end()) + { + pathIt = pathTree.insert({ pFilePath, std::list() }).first; + } + + pathIt->second.push_back(entryBlock); + } +} + +//----------------------------------------------------------------------------- +// Purpose: writes the vpk directory tree // Input : hDirectoryFile - -// &vMap - // Output : number of descriptors written //----------------------------------------------------------------------------- -uint64_t VPKDir_t::WriteDescriptor(FileHandle_t hDirectoryFile, - std::map>>& vMap) const +uint64_t VPKDir_t::CTreeBuilder::WriteTree(FileHandle_t hDirectoryFile) const { uint64_t nDescriptors = NULL; - for (auto& iKeyValue : vMap) + for (auto& iKeyValue : m_FileTree) { - FileSystem()->Write(iKeyValue.first.Get(), int(iKeyValue.first.Length() + 1), hDirectoryFile); + FileSystem()->Write(iKeyValue.first.c_str(), int(iKeyValue.first.length() + 1), hDirectoryFile); for (auto& jKeyValue : iKeyValue.second) { - FileSystem()->Write(jKeyValue.first.Get(), int(jKeyValue.first.Length() + 1), hDirectoryFile); + FileSystem()->Write(jKeyValue.first.c_str(), int(jKeyValue.first.length() + 1), hDirectoryFile); for (auto& vEntry : jKeyValue.second) { CUtlString entryPath = vEntry.m_EntryPath.UnqualifiedFilename().StripExtension(); - FileSystem()->Write(entryPath.Get(), int(entryPath.Length() + 1), hDirectoryFile); FileSystem()->Write(&vEntry.m_nFileCRC, sizeof(uint32_t), hDirectoryFile); @@ -1038,37 +1107,6 @@ uint64_t VPKDir_t::WriteDescriptor(FileHandle_t hDirectoryFile, return nDescriptors; } -//----------------------------------------------------------------------------- -// Purpose: builds the vpk directory tree -// Input : &vEntryBlocks - -// &vMap - -//----------------------------------------------------------------------------- -void VPKDir_t::BuildDirectoryTree(const CUtlVector& entryBlocks, - std::map>>& vMap) const -{ - FOR_EACH_VEC(entryBlocks, i) - { - const VPKEntryBlock_t& entryBlock = entryBlocks[i]; - - CUtlString fileExt = entryBlock.m_EntryPath.GetExtension(); - CUtlString filePath = entryBlock.m_EntryPath.DirName(); - - if (!filePath.IsEmpty() && filePath[0] == '.') - { - filePath = " "; // Has to be padded with a space character if empty [root]. - } - if (!vMap.count(fileExt)) - { - vMap.insert({ fileExt, std::map>() }); - } - if (!vMap[fileExt].count(filePath)) - { - vMap[fileExt].insert({ filePath, std::list() }); - } - vMap[fileExt][filePath].push_back(entryBlock); - } -} - //----------------------------------------------------------------------------- // Purpose: builds the vpk directory file // Input : &svDirectoryPath - @@ -1083,11 +1121,11 @@ void VPKDir_t::BuildDirectoryFile(const CUtlString& directoryPath, const CUtlVec return; } - auto vMap = std::map>>(); - BuildDirectoryTree(entryBlocks, vMap); + CTreeBuilder treeBuilder; + treeBuilder.BuildTree(entryBlocks); WriteHeader(hDirectoryFile); - uint64_t nDescriptors = WriteDescriptor(hDirectoryFile, vMap); + uint64_t nDescriptors = treeBuilder.WriteTree(hDirectoryFile); m_Header.m_nDirectorySize = static_cast(FileSystem()->Tell(hDirectoryFile) - sizeof(VPKDirHeader_t)); WriteTreeSize(hDirectoryFile); diff --git a/r5dev/vpklib/packedstore.h b/r5dev/vpklib/packedstore.h index 3b7943fc..f3c34dcb 100644 --- a/r5dev/vpklib/packedstore.h +++ b/r5dev/vpklib/packedstore.h @@ -128,9 +128,22 @@ struct VPKDir_t CUtlVector m_PackFiles; // Vector of pack file names. CUtlString m_DirFilePath; // Path to vpk_dir file. + class CTreeBuilder + { + public: + typedef std::map> PathContainer_t; + typedef std::map TypeContainer_t; + + void BuildTree(const CUtlVector& entryBlocks); + uint64_t WriteTree(FileHandle_t hDirectoryFile) const; + + private: + TypeContainer_t m_FileTree; + }; + VPKDir_t() { - m_Header.m_nHeaderMarker = VPK_HEADER_MARKER; m_Header.m_nMajorVersion = VPK_MAJOR_VERSION; + m_Header.m_nHeaderMarker = VPK_HEADER_MARKER; m_Header.m_nMajorVersion = VPK_MAJOR_VERSION; m_Header.m_nMinorVersion = VPK_MINOR_VERSION; m_Header.m_nDirectorySize = NULL, m_Header.m_nSignatureSize = NULL; m_PackFileCount = NULL; }; @@ -144,9 +157,7 @@ struct VPKDir_t void WriteHeader(FileHandle_t hDirectoryFile) const; void WriteTreeSize(FileHandle_t hDirectoryFile) const; - uint64_t WriteDescriptor(FileHandle_t hDirectoryFile, std::map>>& vMap) const; - void BuildDirectoryTree(const CUtlVector& entryBlocks, std::map>>& vMap) const; void BuildDirectoryFile(const CUtlString& svDirectoryFile, const CUtlVector& entryBlocks); };